{ "mode": "offline-synthetic-planning-diagnostics", "baseline_commit": "bb808e1472d3894e276689115574f126c193c52a", "candidate": "working tree at invocation; published with its source commit", "baseline_source_sha256": { "src/apixly_jev_harness/__init__.py": "10df257ec1770fce69e97a0b5199d9b034092f5557f6a6a1fb2aa61b42e7efe2", "src/apixly_jev_harness/backend.py": "e4b4c80d88c63648c1392f41eb8796430fcbce6f81d1061c71519d0759432eda", "src/apixly_jev_harness/callbacks.py": "2d27ad4540d910354c54de7566546b4c9d99b5a5fa87705a1905885349078497", "src/apixly_jev_harness/cli.py": "8d9434bf8a8b539163194a84e3f099761d08ef32201f99cbce46c139f3726d94", "src/apixly_jev_harness/context.py": "b9d76abb33f8a1ebe8731fe6d5fe6811445132a6046448fc0e2cc871507d50ea", "src/apixly_jev_harness/diagnostics.py": "24950b4d834d9e6c902996c7bcddfa208944e42300ae062e9ebc2a44f75a091f", "src/apixly_jev_harness/engine.py": "19d809ee7af67586b03a472af49908f6ec278231285d16db161a540b3f0247a6", "src/apixly_jev_harness/output.py": "94be8b482e4279e29aca128a35caf54744dbe8db8188f14905488f8ddde93a0b", "src/apixly_jev_harness/surfaces.py": "65ce73edfe9f6a3c16de373b3a1b85671c3d2cc661f3a50dd906a15721c7d8f1", "src/apixly_jev_harness/task.py": "0f5801e9fdb5626e1db65d269c715e8bcf692ebaa3a91f46b9945158b4bbb6db", "src/apixly_jev_harness/task_cli.py": "bc2aeea2af51fe26efc4528efc9f7849078030a1cea1de2420ff1e98dcabf951", "src/apixly_jev_harness/verify.py": "97d2a178902291094f4e16db08732748d234b8c5ad8dd49a55607c8d195f1ed2" }, "candidate_source_sha256": { "src/apixly_jev_harness/__init__.py": "79a4ea23ea1469f218c5d6f4ee8567f0cf12f77dbd658f4afb57daa90c2a3039", "src/apixly_jev_harness/backend.py": "0ddfab6b270d8219a66614630e70aa10a74cb531f8075d07c255df145725ed5f", "src/apixly_jev_harness/callbacks.py": "2d27ad4540d910354c54de7566546b4c9d99b5a5fa87705a1905885349078497", "src/apixly_jev_harness/cli.py": "8d9434bf8a8b539163194a84e3f099761d08ef32201f99cbce46c139f3726d94", "src/apixly_jev_harness/context.py": "b9d76abb33f8a1ebe8731fe6d5fe6811445132a6046448fc0e2cc871507d50ea", "src/apixly_jev_harness/diagnostics.py": "9e8e98dc4b103c1a6687dbd31bd2dd01739309c7e4021b9ecb0f28b6976a0f71", "src/apixly_jev_harness/engine.py": "19d809ee7af67586b03a472af49908f6ec278231285d16db161a540b3f0247a6", "src/apixly_jev_harness/output.py": "94be8b482e4279e29aca128a35caf54744dbe8db8188f14905488f8ddde93a0b", "src/apixly_jev_harness/surfaces.py": "65ce73edfe9f6a3c16de373b3a1b85671c3d2cc661f3a50dd906a15721c7d8f1", "src/apixly_jev_harness/task.py": "0f5801e9fdb5626e1db65d269c715e8bcf692ebaa3a91f46b9945158b4bbb6db", "src/apixly_jev_harness/task_cli.py": "9cf30e5ed50680ac3b0212bdd8118d67f4272cc7690941a6e57e48fd2ccf850d", "src/apixly_jev_harness/verify.py": "97d2a178902291094f4e16db08732748d234b8c5ad8dd49a55607c8d195f1ed2" }, "runner_sha256": "e21758013f7cb1ece4a5b8b871dd4b5c8e7ee8e9c5a3f9ab893e68901cee18cc", "python_version": "3.14.2", "cases": { "direct_ascii_60k": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "direct_chinese_60k": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "direct_254_choices": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "direct_missing_record_field": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "direct_small_mock_choice": { "mode": "direct", "mock_response": "choice" }, "loop_tool_oversize": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "loop_parameter_oversize": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "prior_unknown_then_parameter": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "fallback_then_effect_error": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "unknown_plan_deferred": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "unsafe_plan_enum_injection": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "generic_spec_value_error": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "mock_http_413_loop": { "mode": "loop", "mock_response": "http_413" }, "mock_http_413_direct": { "mode": "direct", "mock_response": "http_413" } }, "behavior_parity": true, "reproduction": "python benchmarks/planning_diagnostics.py --output local-results/planning-diagnostics-ab.json", "scope": "Fourteen fixed synthetic fixtures, three paired repeats with alternating arm order. Actual pinned core.plan handles oversized ASCII/Chinese records, 254 options, required-record fields and dependent parameter descriptions. Small/HTTP/retry fixtures execute the pinned typed batch and provider Client only through explicit httpx.MockTransport with a synthetic key; real HTTPTransport is forbidden. Mock dispatches and HTTP attempts are separate from actual model usage, which is zero if the guard is never hit. Two injected plan fixtures test safe enum projection; a generic 17,000-character spec ValueError remains an exception. Unknown usage after a mocked retry or HTTP 413 remains unknown in reported mock telemetry.", "negative_results": "New fixed planning metadata and reason codes add returned bytes and local projection work. Request fit limits, option counts, prompts and loop context limits are unchanged; typed NEEDS_CONTEXT is preserved instead of being folded into NEEDS_NARROWING, and unrecognized statuses become UNAVAILABLE. End-state/selection/effect parity excludes that deliberate typed-status correction and diagnostic fields. No semantic quality, speed, real-provider fit, live HTTP behavior, fee benefit or automatic retry improvement is claimed. The only retry is the pinned SDK's existing bounded policy exercised by synthetic MockTransport. A trusted fallback clears the current failure diagnostic but keeps its historical call metadata; later effects remain execution_uncertain.", "cost_conversion": "No tariff or savings assumed. Actual model usage is zero only with no forbidden real transport attempts; a guard hit records unknown actual usage and fails the fixture. Reported mock usage and unknown mock attempts never represent paid usage. Real whole-operation timing includes synthetic registration, actual planning, mocked typed batch/provider validation, bounded mock transport attempts, permitted effect and private journal serialization; excludes process/import/client-fixture setup and terminal-analysis/source hashing. Returned context bytes exclude local archive paths; raw candidate bodies, injected detail and response text are omitted.", "summary": { "baseline": { "runs": 42, "correct": 12, "behavior_correct": 42, "planning_correct": 12, "diagnostic_correct": 33, "typed_status_correct": 33, "mock_batch_dispatches": 18, "mock_http_attempts": 21, "effects": 3, "intent_events": 3, "real_network_attempts": 0, "raw_marker_exposures": 0, "median_whole_operation_seconds": 0.002040020495769568, "returned_context_bytes": 33295, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true } }, "candidate": { "runs": 42, "correct": 42, "behavior_correct": 42, "planning_correct": 42, "diagnostic_correct": 42, "typed_status_correct": 42, "mock_batch_dispatches": 18, "mock_http_attempts": 21, "effects": 3, "intent_events": 3, "real_network_attempts": 0, "raw_marker_exposures": 0, "median_whole_operation_seconds": 0.0020040209928993136, "returned_context_bytes": 37777, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true } } }, "rows": [ { "case": "direct_ascii_60k", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0015078750002430752, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "direct_ascii_60k", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002654708005138673, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "direct_ascii_60k", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0011460420064395294, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "direct_ascii_60k", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0026622919976944104, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "direct_ascii_60k", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0011713330022757873, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "direct_ascii_60k", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "A", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0022605410049436614, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "direct_chinese_60k", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0017705000063870102, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "direct_chinese_60k", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001354541993350722, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "direct_chinese_60k", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.00235574999533128, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "direct_chinese_60k", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0013082079967716709, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "direct_chinese_60k", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0026687500067055225, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "direct_chinese_60k", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "character": "\u4e2d", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0020736669976031408, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "direct_254_choices", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 254, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 254 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 254 }, "source_count": 254 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.00219462500535883, "returned_context_bytes": 1896, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "direct_254_choices", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 254, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 254 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 254 }, "source_count": 254 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.003182542001013644, "returned_context_bytes": 2045, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "direct_254_choices", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 254, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 254 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 254 }, "source_count": 254 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0017458329966757447, "returned_context_bytes": 2045, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "direct_254_choices", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 254, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 254 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 254 }, "source_count": 254 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.002716457995120436, "returned_context_bytes": 1896, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "direct_254_choices", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 254, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 254 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 254 }, "source_count": 254 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0016605419950792566, "returned_context_bytes": 1896, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "direct_254_choices", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "records": 254, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 254, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 254 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 254 }, "source_count": 254 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0017830839933594689, "returned_context_bytes": 2045, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "direct_missing_record_field", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0010029999975813553, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "direct_missing_record_field", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_CONTEXT": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_context" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001676375002716668, "returned_context_bytes": 385, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "direct_missing_record_field", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_CONTEXT": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_context" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0011157499975524843, "returned_context_bytes": 385, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "direct_missing_record_field", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0029814580047968775, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "direct_missing_record_field", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0010848749952856451, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "direct_missing_record_field", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "required_record_fields": [ "source.required" ], "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_CONTEXT": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_context" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0008291249978356063, "returned_context_bytes": 385, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "direct_small_mock_choice", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "choice" }, "expected": { "status": "admitted", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "admitted", "exception_type": null, "complete": true, "selected_count": 1, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "OK": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0020724579953821376, "returned_context_bytes": 835, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "direct_small_mock_choice", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "choice" }, "expected": { "status": "admitted", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "admitted", "exception_type": null, "complete": true, "selected_count": 1, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "OK": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0012732920004054904, "returned_context_bytes": 835, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "direct_small_mock_choice", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "choice" }, "expected": { "status": "admitted", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "admitted", "exception_type": null, "complete": true, "selected_count": 1, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "OK": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001301915996009484, "returned_context_bytes": 835, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "direct_small_mock_choice", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "choice" }, "expected": { "status": "admitted", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "admitted", "exception_type": null, "complete": true, "selected_count": 1, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "OK": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002146958009689115, "returned_context_bytes": 835, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "direct_small_mock_choice", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "choice" }, "expected": { "status": "admitted", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "admitted", "exception_type": null, "complete": true, "selected_count": 1, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "OK": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001578458002768457, "returned_context_bytes": 835, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "direct_small_mock_choice", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "choice" }, "expected": { "status": "admitted", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "admitted", "exception_type": null, "complete": true, "selected_count": 1, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "OK": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0015202920039882883, "returned_context_bytes": 835, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "loop_tool_oversize", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "NEEDS_NARROWING": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 4 }, "source_count": 4 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.003307292005047202, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "loop_tool_oversize", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "NEEDS_NARROWING": 4 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 4 }, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0031136670004343614, "returned_context_bytes": 841, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "loop_tool_oversize", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "NEEDS_NARROWING": 4 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 4 }, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.003281541998148896, "returned_context_bytes": 839, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "loop_tool_oversize", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "NEEDS_NARROWING": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 4 }, "source_count": 4 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.002007582996156998, "returned_context_bytes": 688, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "loop_tool_oversize", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "NEEDS_NARROWING": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 4 }, "source_count": 4 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0027126249915454537, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "loop_tool_oversize", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "tool", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "NEEDS_NARROWING": 4 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 4 }, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0027865419979207218, "returned_context_bytes": 839, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "loop_parameter_oversize", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2687, "request_json_utf8_bytes": 2687, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.002736292008194141, "returned_context_bytes": 1626, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "loop_parameter_oversize", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.006056749989511445, "returned_context_bytes": 1776, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "loop_parameter_oversize", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0033659999899100512, "returned_context_bytes": 1777, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "loop_parameter_oversize", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0025123749946942553, "returned_context_bytes": 1624, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "loop_parameter_oversize", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0026291670073987916, "returned_context_bytes": 1626, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "loop_parameter_oversize", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0026914999907603487, "returned_context_bytes": 1777, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "prior_unknown_then_parameter", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "usage_complete": false, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "mock_request_sizes": [ { "request_json_characters": 2686, "request_json_utf8_bytes": 2686, "question_count": 1, "criterion_count": 6 }, { "request_json_characters": 2686, "request_json_utf8_bytes": 2686, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0037216250057099387, "returned_context_bytes": 1548, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "prior_unknown_then_parameter", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "usage_complete": false, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 }, { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002934958989499137, "returned_context_bytes": 1703, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "prior_unknown_then_parameter", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "usage_complete": false, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 }, { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0043727919983211905, "returned_context_bytes": 1702, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "prior_unknown_then_parameter", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "usage_complete": false, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 }, { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0033062500006053597, "returned_context_bytes": 1551, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "prior_unknown_then_parameter", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "usage_complete": false, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 }, { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "matches_expected": false }, "diagnostic_correct": false, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0027609170065261424, "returned_context_bytes": 1549, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "prior_unknown_then_parameter", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "parameter", "text_repetitions": 60000, "mock_response": "retry_then_choice", "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "usage_complete": false, "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 2, "mock_request_sizes": [ { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 }, { "request_json_characters": 2688, "request_json_utf8_bytes": 2688, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": { "stage": "parameter_selection", "reason_codes": [ "request_needs_narrowing" ], "automatic_retry": false, "tool_id": "inspect", "parameter": "target" }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002880291998735629, "returned_context_bytes": 1703, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "fallback_then_effect_error", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "execution_uncertain", "effects": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "execution_uncertain", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 1, "intent_events": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2125, "request_json_utf8_bytes": 2125, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0030075410031713545, "returned_context_bytes": 1485, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "fallback_then_effect_error", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "execution_uncertain", "effects": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "execution_uncertain", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 1, "intent_events": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2126, "request_json_utf8_bytes": 2126, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002689832996111363, "returned_context_bytes": 1633, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "fallback_then_effect_error", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "execution_uncertain", "effects": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "execution_uncertain", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 1, "intent_events": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2126, "request_json_utf8_bytes": 2126, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002639875005115755, "returned_context_bytes": 1633, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "fallback_then_effect_error", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "execution_uncertain", "effects": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "execution_uncertain", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 1, "intent_events": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2126, "request_json_utf8_bytes": 2126, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.002743124990956858, "returned_context_bytes": 1485, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "fallback_then_effect_error", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "execution_uncertain", "effects": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "execution_uncertain", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 1, "intent_events": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2125, "request_json_utf8_bytes": 2125, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.004348458998720162, "returned_context_bytes": 1485, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "fallback_then_effect_error", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "location": "fallback", "text_repetitions": 60000, "plan_code": "request_needs_narrowing", "typed_status": "NEEDS_NARROWING" }, "expected": { "status": "execution_uncertain", "effects": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": true, "diagnostic": null }, "result": { "status": "execution_uncertain", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 0, "effects": 1, "intent_events": 1, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2126, "request_json_utf8_bytes": 2126, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "OK": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true }, { "stage": "parameter_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_narrowing" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 }, { "items": 0, "deferred_status_counts": { "NEEDS_NARROWING": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 5, "output_tokens": 1 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.003129542004899122, "returned_context_bytes": 1634, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "unknown_plan_deferred", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "UNRECOGNIZED": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0007546659908257425, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "unknown_plan_deferred", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_planning_review" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "UNRECOGNIZED": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0011167080083396286, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "unknown_plan_deferred", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_planning_review" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "UNRECOGNIZED": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0007550000009359792, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "unknown_plan_deferred", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "UNRECOGNIZED": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0008060410036705434, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "unknown_plan_deferred", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "UNRECOGNIZED": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0007622920093126595, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "unknown_plan_deferred", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "unknown", "plan_code": "request_planning_review", "typed_status": "UNAVAILABLE" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_planning_review" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "UNRECOGNIZED": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0010197499941568822, "returned_context_bytes": 387, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "unsafe_plan_enum_injection", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.001181624989840202, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "unsafe_plan_enum_injection", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_CONTEXT": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_context" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001775249998900108, "returned_context_bytes": 385, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "unsafe_plan_enum_injection", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_CONTEXT": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_context" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001266374994884245, "returned_context_bytes": 385, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "unsafe_plan_enum_injection", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0007857910095481202, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "unsafe_plan_enum_injection", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_NARROWING": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": false } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": false, "typed_status_correct": false, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": false, "whole_operation_seconds": 0.0007378330046776682, "returned_context_bytes": 238, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "unsafe_plan_enum_injection", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "plan_stub": "injected", "plan_code": "request_needs_context", "typed_status": "NEEDS_CONTEXT" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": true, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "NEEDS_CONTEXT": 2 }, "planning": { "scope": "current_selection", "phase": "request_planning", "inference_dispatched": false, "reason_codes": [ "request_needs_context" ] }, "planning_raw_marker_exposures": 0, "requests_reported": 0, "usage_complete": true, "usage_known": true, "provider_failure_codes": [], "raw_planning_matches_expected": true } ], "plans": [ { "items": 0, "deferred_status_counts": { "NEEDS_CONTEXT": 2 }, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": true, "usage_known": true, "reported_mock_usage": { "input_tokens": 0, "output_tokens": 0 }, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0012437080004019663, "returned_context_bytes": 385, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "generic_spec_value_error", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "expected": { "status": "raised_value_error", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": null, "diagnostic": null }, "result": { "status": "raised_value_error", "exception_type": "ValueError", "complete": null, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [], "plans": [], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": null, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0010591669997666031, "returned_context_bytes": 32, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "generic_spec_value_error", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "expected": { "status": "raised_value_error", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": null, "diagnostic": null }, "result": { "status": "raised_value_error", "exception_type": "ValueError", "complete": null, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [], "plans": [], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": null, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0013340830046217889, "returned_context_bytes": 32, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "generic_spec_value_error", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "expected": { "status": "raised_value_error", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": null, "diagnostic": null }, "result": { "status": "raised_value_error", "exception_type": "ValueError", "complete": null, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [], "plans": [], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": null, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0008446249994449317, "returned_context_bytes": 32, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "generic_spec_value_error", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "expected": { "status": "raised_value_error", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": null, "diagnostic": null }, "result": { "status": "raised_value_error", "exception_type": "ValueError", "complete": null, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [], "plans": [], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": null, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0012756659998558462, "returned_context_bytes": 32, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "generic_spec_value_error", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "expected": { "status": "raised_value_error", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": null, "diagnostic": null }, "result": { "status": "raised_value_error", "exception_type": "ValueError", "complete": null, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [], "plans": [], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": null, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0007989169971551746, "returned_context_bytes": 32, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "generic_spec_value_error", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "context_repetitions": 17000, "expected_exception": "ValueError" }, "expected": { "status": "raised_value_error", "effects": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "usage_complete": null, "diagnostic": null }, "result": { "status": "raised_value_error", "exception_type": "ValueError", "complete": null, "selected_count": 0, "review_count": 0, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 0, "mock_http_attempts": 0, "mock_request_sizes": [], "planning_calls": [], "plans": [], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": null, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0007248750043800101, "returned_context_bytes": 32, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "mock_http_413_loop", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "mock_response": "http_413" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2250, "request_json_utf8_bytes": 2250, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "UNAVAILABLE": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002652291994309053, "returned_context_bytes": 1108, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "mock_http_413_loop", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "mock_response": "http_413" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2250, "request_json_utf8_bytes": 2250, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "UNAVAILABLE": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0022068750113248825, "returned_context_bytes": 1108, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "mock_http_413_loop", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "mock_response": "http_413" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2250, "request_json_utf8_bytes": 2250, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "UNAVAILABLE": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.005559291996178217, "returned_context_bytes": 1108, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "mock_http_413_loop", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "mock_response": "http_413" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2250, "request_json_utf8_bytes": 2250, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "UNAVAILABLE": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0024600000033387914, "returned_context_bytes": 1108, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "mock_http_413_loop", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "mock_response": "http_413" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2250, "request_json_utf8_bytes": 2250, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "UNAVAILABLE": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002182625001296401, "returned_context_bytes": 1108, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "mock_http_413_loop", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "loop", "mock_response": "http_413" }, "expected": { "status": "needs_review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false } }, "result": { "status": "needs_review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 4, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 2250, "request_json_utf8_bytes": 2250, "question_count": 1, "criterion_count": 6 } ], "planning_calls": [ { "stage": "tool_selection", "typed_status_counts": { "UNAVAILABLE": 4 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 4 } ], "diagnostic": { "stage": "tool_selection", "reason_codes": [ "http_413_body_suppressed" ], "automatic_retry": false }, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0022642499970970675, "returned_context_bytes": 1108, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 }, { "case": "mock_http_413_direct", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "http_413" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0015105420025065541, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 1 }, { "case": "mock_http_413_direct", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "http_413" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0019343749881954864, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 1 }, { "case": "mock_http_413_direct", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "http_413" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.002594749996205792, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 2 }, { "case": "mock_http_413_direct", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "http_413" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.001359625006443821, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 2 }, { "case": "mock_http_413_direct", "runtime_version": "0.1.9", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "http_413" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0015103339974302799, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "baseline", "repeat": 3 }, { "case": "mock_http_413_direct", "runtime_version": "0.1.10", "runtime_loaded_from_requested_source": true, "fixture": { "mode": "direct", "mock_response": "http_413" }, "expected": { "status": "review", "effects": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "usage_complete": false, "diagnostic": null }, "result": { "status": "review", "exception_type": null, "complete": false, "selected_count": 0, "review_count": 2, "effects": 0, "intent_events": 0, "mock_batch_dispatches": 1, "mock_http_attempts": 1, "mock_request_sizes": [ { "request_json_characters": 638, "request_json_utf8_bytes": 638, "question_count": 1, "criterion_count": 4 } ], "planning_calls": [ { "stage": "direct_selection", "typed_status_counts": { "UNAVAILABLE": 2 }, "planning": null, "planning_raw_marker_exposures": 0, "requests_reported": 1, "usage_complete": false, "usage_known": false, "provider_failure_codes": [ "http_413_body_suppressed" ], "raw_planning_matches_expected": true } ], "plans": [ { "items": 1, "deferred_status_counts": {}, "source_count": 2 } ], "diagnostic": null, "diagnostic_correct": true, "planning_correct": true, "typed_status_correct": true, "behavior_correct": true, "usage_complete": false, "usage_known": false, "reported_mock_usage": null, "real_network_attempts": 0, "raw_marker_exposures": 0 }, "correct": true, "whole_operation_seconds": 0.0014559999981429428, "returned_context_bytes": 689, "actual_model_usage": { "requests": 0, "models": {}, "input_tokens": 0, "output_tokens": 0, "usage_complete": true }, "arm": "candidate", "repeat": 3 } ] }