{ "schema_version": 1, "kind": "live-same-output-selection-speed", "version": "0.4.1", "timestamp_utc": "2026-10-04T10:40:23.039439+00:00", "platform": "Darwin", "python": "3.14.2", "records": 96, "questions_per_record": 2, "records_context_bytes": 12351, "fixture_sha256": "ec64a2486fdb0e2603822105da093d3a96a5c4da8ebeeb1ca2dcc0fb1a3ef2ec", "expected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "expected_ids_sha256": "3ef4a998a3dd603e0b5f23ba3c025df08fbe63cff8092720bbb7915fbe2715ac", "task": "Select current connection-establishment failures before any HTTP response. Exclude recovered history and post-response errors.", "requirements": [ { "id": "establishment", "statement": "The CURRENT request fails during DNS, TCP or TLS establishment, not a historical attempt.", "expected": true }, { "id": "response", "statement": "The CURRENT request received HTTP response headers.", "expected": false } ], "primary_models": [ "gpt-5.6-luna", "gpt-6-astra" ], "primary_reasoning_effort": "medium", "repetitions": 3, "result_cache": false, "concurrency_cap": 30, "requested_jev_workers": 12, "order_policy": "Rotate the three arms across repetitions; execute each run sequentially.", "scope": "96 synthetic records (default), two atomic semantic conditions, same complete ID-set output. Jev auto-batched API filtering versus signed-in Codex CLI directly screening the same supplied evidence with no tools. Prepared fixture directories excluded and timed separately. Jev includes planning, transport, typed parsing and program summary; Codex includes prompt serialization, process startup, provider/agent turn and final response. No primary continuation follows Jev in this stage benchmark.", "latency_caveat": "Execution-path screening latency, not an isolated native-model/API comparison or a whole-agent speedup. Native inference latency and CLI process overhead are not independently observable. Provider caching may occur; there is no inference-result cache. Three repetitions are a smoke sample, not a confidence interval or broad workload guarantee.", "billing_caveat": "Actual token usage is measured when available. Codex uses signed-in subscription authentication; actual invoiced billing is unavailable. Unknown failures remain null, never assumed free.", "runs": [ { "arm": "jev", "requested_model": "jev-1.13.0", "observed_models": [ "jev-1.13.0" ], "elapsed_ms": 1238.03, "pool_elapsed_ms": 1150, "provider_request_elapsed_ms": [ null, null ], "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 93226, "returned_context_bytes": 375, "logical_requests": 2, "physical_requests": 2, "effective_workers": 2, "review_record_ids": [], "failed_requests": 0, "unknown_usage_attempts": 0, "usage": { "input_tokens": 30362, "output_tokens": 9182 }, "known_usage": { "input_tokens": 30362, "output_tokens": 9182 }, "usage_complete": true, "actual_billed_usd": null, "timeout": false, "tool_items": 0, "exit_code": null, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 0 }, { "arm": "gpt-5.6-luna", "requested_model": "gpt-5.6-luna", "observed_models": [], "resolved_model_observation": "Requested CLI model; provider-resolved model ID unavailable.", "elapsed_ms": 13421.326, "fixture_setup_ms": 0.233, "pool_elapsed_ms": null, "provider_request_elapsed_ms": null, "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 13224, "output_schema_bytes": 196, "returned_context_bytes": 375, "logical_requests": 1, "physical_requests": null, "failed_requests": 0, "unknown_usage_attempts": 0, "actual_billed_usd": null, "timeout": false, "exit_code": 0, "tool_items": 0, "usage": { "input_tokens": 19684, "cached_input_tokens": 0, "output_tokens": 258 }, "known_usage": { "input_tokens": 19684, "cached_input_tokens": 0, "output_tokens": 258 }, "usage_complete": true, "turn_count": 1, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 0 }, { "arm": "gpt-6-astra", "requested_model": "gpt-6-astra", "observed_models": [], "resolved_model_observation": "Requested CLI model; provider-resolved model ID unavailable.", "elapsed_ms": 14543.075, "fixture_setup_ms": 0.474, "pool_elapsed_ms": null, "provider_request_elapsed_ms": null, "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 13224, "output_schema_bytes": 196, "returned_context_bytes": 375, "logical_requests": 1, "physical_requests": null, "failed_requests": 0, "unknown_usage_attempts": 0, "actual_billed_usd": null, "timeout": false, "exit_code": 0, "tool_items": 0, "usage": { "input_tokens": 22072, "cached_input_tokens": 0, "output_tokens": 164 }, "known_usage": { "input_tokens": 22072, "cached_input_tokens": 0, "output_tokens": 164 }, "usage_complete": true, "turn_count": 1, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 0 }, { "arm": "gpt-5.6-luna", "requested_model": "gpt-5.6-luna", "observed_models": [], "resolved_model_observation": "Requested CLI model; provider-resolved model ID unavailable.", "elapsed_ms": 13931.801, "fixture_setup_ms": 0.511, "pool_elapsed_ms": null, "provider_request_elapsed_ms": null, "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 13224, "output_schema_bytes": 196, "returned_context_bytes": 375, "logical_requests": 1, "physical_requests": null, "failed_requests": 0, "unknown_usage_attempts": 0, "actual_billed_usd": null, "timeout": false, "exit_code": 0, "tool_items": 0, "usage": { "input_tokens": 19684, "cached_input_tokens": 0, "output_tokens": 282 }, "known_usage": { "input_tokens": 19684, "cached_input_tokens": 0, "output_tokens": 282 }, "usage_complete": true, "turn_count": 1, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 1 }, { "arm": "gpt-6-astra", "requested_model": "gpt-6-astra", "observed_models": [], "resolved_model_observation": "Requested CLI model; provider-resolved model ID unavailable.", "elapsed_ms": 14332.711, "fixture_setup_ms": 0.33, "pool_elapsed_ms": null, "provider_request_elapsed_ms": null, "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 13224, "output_schema_bytes": 196, "returned_context_bytes": 375, "logical_requests": 1, "physical_requests": null, "failed_requests": 0, "unknown_usage_attempts": 0, "actual_billed_usd": null, "timeout": false, "exit_code": 0, "tool_items": 0, "usage": { "input_tokens": 22072, "cached_input_tokens": 0, "output_tokens": 164 }, "known_usage": { "input_tokens": 22072, "cached_input_tokens": 0, "output_tokens": 164 }, "usage_complete": true, "turn_count": 1, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 1 }, { "arm": "jev", "requested_model": "jev-1.13.0", "observed_models": [ "jev-1.13.0" ], "elapsed_ms": 1313.291, "pool_elapsed_ms": 1260, "provider_request_elapsed_ms": [ null, null ], "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 93226, "returned_context_bytes": 375, "logical_requests": 2, "physical_requests": 2, "effective_workers": 2, "review_record_ids": [], "failed_requests": 0, "unknown_usage_attempts": 0, "usage": { "input_tokens": 30362, "output_tokens": 9182 }, "known_usage": { "input_tokens": 30362, "output_tokens": 9182 }, "usage_complete": true, "actual_billed_usd": null, "timeout": false, "tool_items": 0, "exit_code": null, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 1 }, { "arm": "gpt-6-astra", "requested_model": "gpt-6-astra", "observed_models": [], "resolved_model_observation": "Requested CLI model; provider-resolved model ID unavailable.", "elapsed_ms": 15592.549, "fixture_setup_ms": 0.297, "pool_elapsed_ms": null, "provider_request_elapsed_ms": null, "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 13224, "output_schema_bytes": 196, "returned_context_bytes": 375, "logical_requests": 1, "physical_requests": null, "failed_requests": 0, "unknown_usage_attempts": 0, "actual_billed_usd": null, "timeout": false, "exit_code": 0, "tool_items": 0, "usage": { "input_tokens": 22072, "cached_input_tokens": 0, "output_tokens": 164 }, "known_usage": { "input_tokens": 22072, "cached_input_tokens": 0, "output_tokens": 164 }, "usage_complete": true, "turn_count": 1, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 2 }, { "arm": "jev", "requested_model": "jev-1.13.0", "observed_models": [ "jev-1.13.0" ], "elapsed_ms": 1375.407, "pool_elapsed_ms": 1296, "provider_request_elapsed_ms": [ null, null ], "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 93226, "returned_context_bytes": 375, "logical_requests": 2, "physical_requests": 2, "effective_workers": 2, "review_record_ids": [], "failed_requests": 0, "unknown_usage_attempts": 0, "usage": { "input_tokens": 30362, "output_tokens": 9182 }, "known_usage": { "input_tokens": 30362, "output_tokens": 9182 }, "usage_complete": true, "actual_billed_usd": null, "timeout": false, "tool_items": 0, "exit_code": null, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 2 }, { "arm": "gpt-5.6-luna", "requested_model": "gpt-5.6-luna", "observed_models": [], "resolved_model_observation": "Requested CLI model; provider-resolved model ID unavailable.", "elapsed_ms": 13209.523, "fixture_setup_ms": 0.653, "pool_elapsed_ms": null, "provider_request_elapsed_ms": null, "native_provider_inference_ms": null, "process_overhead_ms": null, "input_context_bytes": 13224, "output_schema_bytes": 196, "returned_context_bytes": 375, "logical_requests": 1, "physical_requests": null, "failed_requests": 0, "unknown_usage_attempts": 0, "actual_billed_usd": null, "timeout": false, "exit_code": 0, "tool_items": 0, "usage": { "input_tokens": 19684, "cached_input_tokens": 0, "output_tokens": 248 }, "known_usage": { "input_tokens": 19684, "cached_input_tokens": 0, "output_tokens": 248 }, "usage_complete": true, "turn_count": 1, "answer_valid": true, "selected_ids": [ "r000", "r001", "r002", "r006", "r008", "r009", "r010", "r014", "r016", "r017", "r018", "r022", "r024", "r025", "r026", "r030", "r032", "r033", "r034", "r038", "r040", "r041", "r042", "r046", "r048", "r049", "r050", "r054", "r056", "r057", "r058", "r062", "r064", "r065", "r066", "r070", "r072", "r073", "r074", "r078", "r080", "r081", "r082", "r086", "r088", "r089", "r090", "r094" ], "duplicate_ids": false, "invalid_ids": [], "false_positive_ids": [], "false_negative_ids": [], "needs_review": false, "id_set_correct": true, "complete_correct": true, "repeat": 2 } ], "summary": { "gpt-5.6-luna": { "runs": 3, "complete_correct_runs": 3, "review_runs": 0, "timeout_runs": 0, "median_ms": 13421.326, "mean_ms": 13520.883333333333, "minimum_ms": 13209.523, "maximum_ms": 13931.801, "usage_complete": true }, "gpt-6-astra": { "runs": 3, "complete_correct_runs": 3, "review_runs": 0, "timeout_runs": 0, "median_ms": 14543.075, "mean_ms": 14822.778333333334, "minimum_ms": 14332.711, "maximum_ms": 15592.549, "usage_complete": true }, "jev": { "runs": 3, "complete_correct_runs": 3, "review_runs": 0, "timeout_runs": 0, "median_ms": 1313.291, "mean_ms": 1308.9093333333333, "minimum_ms": 1238.03, "maximum_ms": 1375.407, "usage_complete": true } }, "harness_sha256_at_run": "3c84f0457e24123aa0b3918cb28975efd81d0054515c8f7229870d85170ffbd6", "published_harness_sha256": "c460ac4b78d29949e9f8d2aa157742cd24e034a109b98c184bae7b52ed3cd3a0", "post_run_harness_changes": "Corrected optional per-request latency key for future runs, added sanitized unexpected-harness-failure persistence, unconditional removal of parsed answer on failed subprocess, and automatic harness hash. Completed-run decisions and wall-clock timing were unchanged. Original run did not collect per-request latency; null values remain unaltered.", "runtime_base_commit": "69e319987465cf28134e22cef70d409dcea4b280", "runtime_source_files": { "src/jev_context/analysis.py": "864263de4dff787777dc0f9cdb41071c0dbebd5d41137df7300ed7fdc5fac5b5", "src/jev_context/pool.py": "94f5b622e9f82038113ba695e87ef8c094a47b3f1731f324218bc65e42a007dd", "src/jev_context/provider.py": "675ed25d0c644fe0f8081f6310b07aa24766e1f5df16ccb9d77b43d3c4d1881e", "benchmarks/fixture.py": "8da05e62f000f09b835bedaa29596da9471589b8f82f68ad7e8c25ec68375001" }, "runtime_matches_base_commit": true, "request_latency_measurement_complete": false, "request_latency_measurement_note": "Transport pool elapsed_ms was measured for every Jev run. Individual HTTP request latency_ms was available but not captured by the original harness; request-level nulls are missing measurements, not zero latency. Native inference-only time and Codex process overhead are unavailable for both paths.", "usage_totals_by_arm": { "jev": { "known_usage": { "input_tokens": 91086, "output_tokens": 27546 }, "usage_complete": true, "unknown_usage_attempts": 0, "actual_billed_usd": null }, "gpt-5.6-luna": { "known_usage": { "input_tokens": 59052, "cached_input_tokens": 0, "output_tokens": 788 }, "usage_complete": true, "unknown_usage_attempts": 0, "actual_billed_usd": null }, "gpt-6-astra": { "known_usage": { "input_tokens": 66216, "cached_input_tokens": 0, "output_tokens": 492 }, "usage_complete": true, "unknown_usage_attempts": 0, "actual_billed_usd": null } }, "source_context_note": "records_context_bytes is the common synthetic records JSON. Jev input_context_bytes sums program-built API envelopes (including duplicated typed question schema); primary input_context_bytes is the supplied prompt only, with output schema measured separately. Hidden CLI/system input is not included in supplied bytes; actual model token usage includes it." }