{"schema_version":1,"generated_at":"2026-09-24T18:02:51.173Z","method":{"id":"speedbench-fixed-payload-v19","version":19,"name":"Verified completion time","unit":"ms","payload_bytes":2048,"payload_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","prompt":"This is a text-copying test for measuring response latency. Please reproduce the sample text below verbatim as a single line, including the final period. Your response should contain only the sample text. No tools are needed.\n\nSample text:\nsunrise cabinet cabinet harvest diamond shelter station library evening whisper gallery pattern glacier pattern teacher silence library freedom shelter weather evening teacher balance station station lantern glacier compass orchard balance blossom sunrise station station bicycle whisper gallery freedom diamond teacher thunder station shelter cabinet cabinet harvest teacher channel evening station thunder sunrise crystal fiction teacher library lantern bicycle bicycle evening balance teacher shelter crystal village blossom thunder gallery blossom morning glacier evening lantern compass sunrise freedom horizon lantern whisper glacier compass horizon blossom evening teacher horizon station weather evening diamond thunder blossom picture village pattern emerald shelter bicycle teacher horizon gallery lantern bicycle bicycle sunrise horizon station channel glacier crystal station cabinet crystal cabinet lantern harvest thunder journey horizon gallery pattern evening library morning compass blossom horizon compass diamond thunder pattern balance picture freedom village thunder shelter picture journey crystal weather balance morning glacier diamond horizon picture bicycle glacier crystal picture village diamond shelter compass blossom journey bicycle cabinet bicycle harvest pattern picture emerald picture library diamond shelter freedom bicycle lantern lantern village morning silence harvest sunrise journey orchard whisper sunrise cabinet blossom whisper evening picture harvest thunder shelter morning sunrise channel lantern silence shelter shelter compass morning sunrise emerald channel journey cabinet diamond fiction freedom village glacier station morning channel emerald blossom library harvest blossom lantern cabinet blossom gallery teacher library morning orchard emerald gallery fiction emerald sunrise thunder compass orchard morning crystal blossom channel blossom gallery pattern crystal picture diamond freedom orchard compass orchard horizon sunrise emerald village bicycle evening weather picture weather shelter.","payload":"sunrise cabinet cabinet harvest diamond shelter station library evening whisper gallery pattern glacier pattern teacher silence library freedom shelter weather evening teacher balance station station lantern glacier compass orchard balance blossom sunrise station station bicycle whisper gallery freedom diamond teacher thunder station shelter cabinet cabinet harvest teacher channel evening station thunder sunrise crystal fiction teacher library lantern bicycle bicycle evening balance teacher shelter crystal village blossom thunder gallery blossom morning glacier evening lantern compass sunrise freedom horizon lantern whisper glacier compass horizon blossom evening teacher horizon station weather evening diamond thunder blossom picture village pattern emerald shelter bicycle teacher horizon gallery lantern bicycle bicycle sunrise horizon station channel glacier crystal station cabinet crystal cabinet lantern harvest thunder journey horizon gallery pattern evening library morning compass blossom horizon compass diamond thunder pattern balance picture freedom village thunder shelter picture journey crystal weather balance morning glacier diamond horizon picture bicycle glacier crystal picture village diamond shelter compass blossom journey bicycle cabinet bicycle harvest pattern picture emerald picture library diamond shelter freedom bicycle lantern lantern village morning silence harvest sunrise journey orchard whisper sunrise cabinet blossom whisper evening picture harvest thunder shelter morning sunrise channel lantern silence shelter shelter compass morning sunrise emerald channel journey cabinet diamond fiction freedom village glacier station morning channel emerald blossom library harvest blossom lantern cabinet blossom gallery teacher library morning orchard emerald gallery fiction emerald sunrise thunder compass orchard morning crystal blossom channel blossom gallery pattern crystal picture diamond freedom orchard compass orchard horizon sunrise emerald village bicycle evening weather picture weather shelter.","system_prompt":"Answer the user. Do not use tools. Do not add a preamble.","normalization":"Remove at most one terminal LF or CRLF; every other UTF-8 byte must match.","clock":"Windows QueryPerformanceCounter via System.Diagnostics.Stopwatch; integer ticks and frequency retained","start_boundary":"Immediately before CreateProcessW(CREATE_SUSPENDED), after job, pipe and helper setup","stop_boundary":"After root process exit, stdout and stderr closure, and the Windows Job Object reports zero active processes","statistic":"Elapsed time of the first verified successful attempt; at most three attempts. No average or fastest-attempt selection.","execution":"One child at a time in roster order; stop each model on its first verified success; retain every failed attempt and retry delay.","trials_per_model":3,"effort":"high","response_model_aliases":{"grok-4.6":["grok-4.6","grok-4.6-build"],"grok-4.7":["grok-4.7","grok-4.7-build"]},"timeout_ms":300000,"description":"Elapsed subscription-CLI time to return an exactly verified 2 KiB payload with complete process-tree containment. Lower is faster. Includes process creation, job assignment/resume, network, reasoning, output, and shutdown; excludes helper setup.","limitations":["A synthetic copying workload, not general coding performance or raw model generation throughput.","One account per provider on one machine. Caching, routing, and provider load can affect results.","The same named high effort does not imply equal reasoning budgets across providers.","One verified successful attempt describes this batch; failed attempts and retry waits remain evidence. It does not establish a statistically significant winner.","Codex identifies the requested pin through its authenticated catalog; its JSON stream does not attest the serving model.","Collection v2 currently supports Windows only; v1 measurements remain a separate historical method.","The collector records local clock evidence and process containment; it is not third-party attestation or a provider-side compute measurement.","Devin uses its native system context, a fresh workspace and explicit configuration with external imports disabled. Its available rules/skills inventory is retained privately; Windows home discovery can still list user skills. This is CLI completion time, not identical system prompts across providers.","Devin export writing and native context startup are included in process completion time; export parsing happens after the clock. Exported model identity is CLI evidence, not independent serving-model attestation.","Devin injects a native available-skills catalog, including Windows user skill descriptions, even with configuration imports disabled. No skill is invoked. Native context differs across CLIs and is retained for audit.","Antigravity native context and built-in inventory remain private. CLI model initialization is not server attestation. Provider model-call count remains unknown, including native checkpoint activity.","Cursor Fable is the requested selection and may fall back to Opus. The SDK does not attest which model served the response.","Version 8 uses a revised instruction with the unchanged 2,048-byte payload. Do not pool timings with earlier prompt versions.","Version 14 scores the Cursor Grok 4.7 High route at 256K context. Cursor's registry rejected the advertised 500K non-fast variant in both the v12 rollout and the reviewed 2026-09-21 diagnostic, so the verified routable 256K, high, non-fast pin is measured instead. Do not pool its timings with other model, context or prompt versions.","Version 15 adds Claude Opus 5.5 through the Cursor SDK at high effort, 300K context and normal (non-fast) speed. Cursor exposes no separate thinking switch for this model. Do not pool its timings with other model, context or prompt versions.","Version 16 adds Claude Opus 5.5 through the native Claude Code CLI on the lab owner claude.ai subscription at high effort. Thinking cannot be disabled on this model. The CLI pin, API response model and terminal per-model usage identify the model; this is not server attestation. Claude Code always loads its built-in agents-md and telemetry plugins. Do not pool its timings with the Cursor route or other recipes.","Version 17 measures GPT-6 Sol and GPT-6 Luna instead of GPT-5.6 Sol and Luna, drops the superseded Grok 4.6 (Build and Cursor) and Cursor Opus 5 routes, and pins Antigravity CLI 1.2.8. Do not pool its timings with earlier rosters.","Version 18 measures Claude only through the native Claude Code subscription (Opus 5.5) and drops the Cursor Opus 5.5 and Cursor Fable 5.1 routes. Fable 5.1 is not measured because its safeguard refuses this payload and a fallback model would answer instead. Do not pool its timings with earlier Cursor Claude routes.","Version 19 is version 18 with Antigravity CLI 1.2.9 pinned after its automatic update; the roster, payload and effort are unchanged. Do not pool its Gemini timings with earlier Antigravity builds."],"capture_version":2,"assessment_version":2,"execution_platforms":["win32"],"process_containment":"Windows Job Object, assigned while suspended, kill-on-job-close, no breakaway","normal_close_drain_ms":50,"normal_close_policy":"After root exit, allow at most 50 ms for natural job/stream teardown; this wait is included in elapsed time. Any remaining descendant requires forced termination and invalidates the trial.","timing_evidence":"QPC start/end/event ticks and frequency; Node monotonic parent envelope; batch ordering on the common QPC clock","response_policy":"One structurally complete provider turn; exact payload; no contradictory text, unknown events, tool calls or unverified capture. Grok modelUsage must report exactly one model call.","targets":[{"provider":"codex","model":"gpt-6-astra","label":"GPT-6 Astra","effort":"high"},{"provider":"codex","model":"gpt-6-sol","label":"GPT-6 Sol","effort":"high"},{"provider":"codex","model":"gpt-5.6-terra","label":"GPT-5.6 Terra","effort":"high"},{"provider":"codex","model":"gpt-6-luna","label":"GPT-6 Luna","effort":"high"},{"provider":"grok","model":"grok-4.7","label":"Grok 4.7","effort":"high"},{"provider":"devin","model":"swe-2-high","label":"SWE-2","effort":"high"},{"provider":"antigravity","model":"gemini-3.8-flash-high","label":"Gemini 3.8 Flash","effort":"high"},{"provider":"claude","model":"claude-opus-5-5","label":"Claude Opus 5.5","effort":"high"},{"provider":"cursor","model":"grok-4.7-256k-high","label":"Grok 4.7 · Cursor","effort":"high"}],"devin_config":{"agent":{"model":"swe-2-high"},"permissions":{"allow":[],"deny":["read","write","edit","exec","grep","glob","fetch","mcp","run_subagent","read_subagent"],"ask":[]},"subagents_enabled":false,"auto_update":false,"notify":"never","read_config_from":{"agents_standard":false,"cursor":false,"windsurf":false,"claude":false,"copilot":false,"opencode":false,"zed":false},"version":1,"shell":{"setup_complete":true},"theme_mode":"nocolor"},"devin_response":"Devin print mode with private ATIF-v1.7 export; exactly one user prompt and one tool-free agent step. Compare stdout with the exported answer. Require the explicit authenticated catalog pin and matching exported model; not server attestation.","devin_account_config":"Copy only devin.org_id from the authenticated local configuration; retain it privately. All other configuration is pinned. Verify configuration before and after capture.","devin_context_policy":"Permit one native available_skills system catalog after the exact user prompt and before the sole agent answer. No external rules, loaded skill instructions, tool calls or observations. Retain full native context privately.","devin_usage":"Require integer prompt/completion counters and matching final totals. An omitted agent cached_tokens counter is zero only when final total_cached_tokens is explicitly zero; reject every other missing or contradictory counter.","antigravity_transport":"One NDJSON user message on stdin, then EOF. Native input proof hashes transmitted bytes; separately verify decoded exact prompt.","antigravity_isolation":"Fresh Windows profile and workspace, native account keyring authentication only; strict permissions with deny rules for files, commands, URLs and MCP. No personal instructions, plugins, hooks, MCP or conversation imports. Disable slash-command expansion. Pin CLI 1.2.9 and reject configuration changes.","antigravity_response":"One matching model initialization, conversation, user turn, completed streamed answer and successful terminal result. Require exact agreement with terminal text and payload; reject tools/subagents, unknown events and missing usage. Retain checkpoint activity separately from answer text.","result_selection":"first_verified_success","max_attempts":3,"retry_delay_ms":5000,"retry_policy":"Persist each reservation before execution and all evidence before retry. Retry safely terminated completed failures; stop on authentication, configuration, storage or unconfirmed termination. Honor longer provider delays within the hourly slot.","cursor_sdk_version":"1.0.31","cursor_sdk_models":{"grok-4.7-256k-high":{"id":"grok-4.7","params":[{"id":"context","value":"256k"},{"id":"reasoning_effort","value":"high"},{"id":"fast","value":"false"}]}},"cursor_sdk_policy":{"tools":[],"agents":{},"mcpServers":{},"local":{"settingSources":[],"enableAgentRetries":false,"customTools":{},"autoReview":false}},"cursor_transport":"One local Cursor SDK send in a contained Node process, exact model and parameters, stdin followed by EOF, retained SDK events. SDK transport and stall retries disabled.","cursor_isolation":"Fresh profile, workspace and JSONL store. Only the expiring user SDK login is copied. No tools, subagents, MCP or ambient settings. Verify the Ultra account has on-demand spending disabled before collection activation.","cursor_response":"One SDK agent and run. Verify exact submitted prompt through retained SDK send argument and independent stdin byte delivery proof; any optional user echo must match. Concatenated assistant deltas and exported conversation must match the terminal answer, with terminal success, matching SDK model identity and complete usage. Client evidence, not server attestation; provider model calls unknown.","cursor_fable_routing":"requested_selection_with_possible_opus_fallback","codex_instruction_transport":"Fresh model_instructions_file with the frozen system prompt, verified before and after capture; strict configuration, no ambient skills or environment context.","devin_context_validation":"Accept one native available_skills catalog with system/unknown telemetry and omitted per-step cached_tokens only when the final total is explicitly zero, as in v9. Preserve v10 rejection behavior for historical replay.","cursor_grok_47_routing":"grok-4.7 with context=256k, reasoning_effort=high, fast=false; exact SDK catalog variant required","cursor_opus_55_routing":"claude-opus-5-5 with context=300k, effort=high, fast=false; exact SDK catalog variant required","claude_transport":"claude -p with --model, explicit --effort high, stream-json output with partial messages, one turn, empty tool list, dontAsk permissions, --safe-mode, no setting sources, disabled slash commands, the frozen system prompt, strict empty MCP configuration and no session persistence. The exact prompt arrives on stdin, then EOF.","claude_isolation":"Fresh empty working folder without inherited CLAUDE.md, AGENTS.md or .claude context. Only the account home login is used: every CLAUDE*, ANTHROPIC*, proxy and inherited session variable is removed; auto-update and nonessential traffic are disabled. Require claude.ai subscription authentication with the first-party API.","claude_response":"One initialization with no tools, MCP servers, skills or slash commands, only the built-in agents-md and telemetry plugins, dontAsk permissions, no API key source and the trial folder. One API message with thinking and text blocks only, matching streamed and final text, one successful end_turn result, no permission denials or subagents, and one per-model usage entry. Reject rate-limited or extra-usage (overage) responses.","antigravity_cli_version":"1.2.9","sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04"},"batches":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6","sequence":366,"started_at":"2026-09-24T18:00:21.915Z","finished_at":"2026-09-24T18:02:51.169Z","updated_at":"2026-09-24T18:02:51.173Z","status":"failed","planned_trials":3,"execution":"serial","method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","context":{"trigger":"scheduled","platform":"win32","arch":"x64","node_version":"v24.13.0","serial_guard":"suite.lock","slot_utc":"2026-09-24T18:00:00.000Z","method_id":"speedbench-fixed-payload-v19","timeout_ms":300000},"revision":41,"models":[{"provider":"codex","model":"gpt-6-astra","label":"GPT-6 Astra","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-1-1","index":1,"status":"passed","started_at":"2026-09-24T18:00:25.891Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:00:39.206Z","elapsed_ms":13314.35,"verified_completion_ms":13314.35,"completion_event_ms":12812.5326,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6079517608403","end_ticks":"6079650751903"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"codex-cli 0.155.1","requested_model":"gpt-6-astra","observed_models":[],"model_verification":{"verified":true,"method":"explicit_cli_pin+authenticated_catalog","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":4048,"cached_input_tokens":0,"output_tokens":262},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"f860f4b23ecffb8124cbaf4d52b2f44f297b5142d90ae7ec06b6b1e653bd5d3c"}],"summary":{"result_ms":13314.35,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-1-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:00:39.299Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"},{"provider":"codex","model":"gpt-6-sol","label":"GPT-6 Sol","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-2-1","index":1,"status":"passed","started_at":"2026-09-24T18:00:41.352Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:00:50.240Z","elapsed_ms":8887.0757,"verified_completion_ms":8887.0757,"completion_event_ms":8510.9187,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6079672226297","end_ticks":"6079761097054"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"codex-cli 0.155.1","requested_model":"gpt-6-sol","observed_models":[],"model_verification":{"verified":true,"method":"explicit_cli_pin+authenticated_catalog","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":4048,"cached_input_tokens":0,"output_tokens":262},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"cb4824618ab993ad2c99504b37d28237d7acb5407c16f80f2334a9c92d8c04ca"}],"summary":{"result_ms":8887.0757,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-2-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:00:50.311Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"},{"provider":"codex","model":"gpt-5.6-terra","label":"GPT-5.6 Terra","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-3-1","index":1,"status":"passed","started_at":"2026-09-24T18:00:52.364Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:01:00.150Z","elapsed_ms":7785.9166,"verified_completion_ms":7785.9166,"completion_event_ms":7407.4553,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6079782338416","end_ticks":"6079860197582"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"codex-cli 0.155.1","requested_model":"gpt-5.6-terra","observed_models":[],"model_verification":{"verified":true,"method":"explicit_cli_pin+authenticated_catalog","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":3425,"cached_input_tokens":0,"output_tokens":262},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"def7834eee6ccd08cccc2a7d737d927c02b19e49f6c1c75e87383bd93cff25fe"}],"summary":{"result_ms":7785.9166,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-3-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:01:00.237Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"},{"provider":"codex","model":"gpt-6-luna","label":"GPT-6 Luna","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-4-1","index":1,"status":"passed","started_at":"2026-09-24T18:01:04.368Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:01:14.264Z","elapsed_ms":9896.0107,"verified_completion_ms":9896.0107,"completion_event_ms":9538.1364,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6079902382904","end_ticks":"6080001343011"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"codex-cli 0.155.1","requested_model":"gpt-6-luna","observed_models":[],"model_verification":{"verified":true,"method":"explicit_cli_pin+authenticated_catalog","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":4048,"cached_input_tokens":2816,"output_tokens":262},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"5cb704ea30e2b4d18a95352d947a62a50b0aac541ddc57c195bd69aa8047f366"}],"summary":{"result_ms":9896.0107,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-4-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:01:14.346Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"},{"provider":"grok","model":"grok-4.7","label":"Grok 4.7","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-5-1","index":1,"status":"passed","started_at":"2026-09-24T18:01:24.901Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:01:38.695Z","elapsed_ms":13793.2686,"verified_completion_ms":13793.2686,"completion_event_ms":13470.9629,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6080107713971","end_ticks":"6080245646657"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"grok 1.0.40 (eb1a2256660d)","requested_model":"grok-4.7","observed_models":["grok-4.7-build"],"model_verification":{"verified":true,"method":"response_model_usage","server_attested":true},"provider_model_calls":1,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":10231,"cache_read_input_tokens":1152,"cache_creation_input_tokens":0,"output_tokens":590,"reasoning_tokens":333},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"9a7ed8f865f52924573b775db1539d5485b7d7ea7a479d5d7364d285f8fc0033"}],"summary":{"result_ms":13793.2686,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-5-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:01:38.811Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"},{"provider":"devin","model":"swe-2-high","label":"SWE-2","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-6-1","index":1,"status":"passed","started_at":"2026-09-24T18:01:45.921Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:01:52.461Z","elapsed_ms":6540.7864,"verified_completion_ms":6540.7864,"completion_event_ms":6540.7864,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6080317909091","end_ticks":"6080383316955"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"devin 3000.10.21 (611c1cba)","requested_model":"swe-2-high","observed_models":["swe-2-high"],"model_verification":{"verified":true,"method":"explicit_cli_pin+authenticated_catalog+atif_export","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"exported_agent_steps":1,"usage":{"input_tokens":11911,"output_tokens":301,"cached_input_tokens":99},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"2245bf3539f957f3e6ed22b2cf26e6e287037422615865820b8269e9ce8b18f6"}],"summary":{"result_ms":6540.7864,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-6-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:01:52.554Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"},{"provider":"antigravity","model":"gemini-3.8-flash-high","label":"Gemini 3.8 Flash","effort":"high","status":"failed","trials":[],"summary":{"result_ms":null,"selected_trial_id":null,"attempt_count":0,"valid_trials":0,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"stop_reason":"collection_stopped","preflight_failure":{"at":"2026-09-24T18:01:57.120Z","phase":"antigravity_preflight"}},{"provider":"claude","model":"claude-opus-5-5","label":"Claude Opus 5.5","effort":"high","status":"failed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-8-1","index":1,"status":"failed","started_at":"2026-09-24T18:01:58.534Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:02:05.104Z","elapsed_ms":6569.9946,"verified_completion_ms":null,"completion_event_ms":6523.6593,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6080444043705","end_ticks":"6080509743651"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"2.1.281 (Claude Code)","requested_model":"claude-opus-5-5","observed_models":["claude-opus-5-5"],"model_verification":{"verified":true,"method":"explicit_cli_pin+stream_init+api_response_model+model_usage","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":2,"output_tokens":691,"cache_read_input_tokens":0,"cache_creation_input_tokens":1199,"reasoning_tokens":0},"validation":{"ok":false,"reasons":["missing_successful_completion","protocol_error"]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"bac234117341165637cb57faeddf7c51e34d64231b912a30d59a338b38a69712"},{"id":"2026-09-24T18-00-21-915Z-1f4155b6-8-2","index":2,"status":"failed","started_at":"2026-09-24T18:02:10.979Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:02:17.780Z","elapsed_ms":6799.9469,"verified_completion_ms":null,"completion_event_ms":6748.4476,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6080568504850","end_ticks":"6080636504319"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"2.1.281 (Claude Code)","requested_model":"claude-opus-5-5","observed_models":["claude-opus-5-5"],"model_verification":{"verified":true,"method":"explicit_cli_pin+stream_init+api_response_model+model_usage","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":2,"output_tokens":691,"cache_read_input_tokens":0,"cache_creation_input_tokens":1200,"reasoning_tokens":0},"validation":{"ok":false,"reasons":["missing_successful_completion","protocol_error"]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"b9f41489af97cb4df2399b05b3b5ce9e715f9e13215ca50695917a237302cc8d"},{"id":"2026-09-24T18-00-21-915Z-1f4155b6-8-3","index":3,"status":"failed","started_at":"2026-09-24T18:02:23.734Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:02:30.251Z","elapsed_ms":6517.0682,"verified_completion_ms":null,"completion_event_ms":6459.3827,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6080696049420","end_ticks":"6080761220102"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"2.1.281 (Claude Code)","requested_model":"claude-opus-5-5","observed_models":["claude-opus-5-5"],"model_verification":{"verified":true,"method":"explicit_cli_pin+stream_init+api_response_model+model_usage","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":2,"output_tokens":691,"cache_read_input_tokens":0,"cache_creation_input_tokens":1201,"reasoning_tokens":0},"validation":{"ok":false,"reasons":["missing_successful_completion","protocol_error"]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"920150cb0ca30326bd2f357bcccddb88c7970f61aeb903258c91d973046837f8"}],"summary":{"result_ms":null,"selected_trial_id":null,"attempt_count":3,"valid_trials":0,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:02:05.167Z","retry":true,"delay_ms":5000,"reason":"retry_completed_failure"},{"after_attempt":2,"at":"2026-09-24T18:02:17.838Z","retry":true,"delay_ms":5000,"reason":"retry_completed_failure"},{"after_attempt":3,"at":"2026-09-24T18:02:30.326Z","retry":false,"delay_ms":0,"reason":"attempts_exhausted"}],"stop_reason":"attempts_exhausted"},{"provider":"cursor","model":"grok-4.7-256k-high","label":"Grok 4.7 · Cursor","effort":"high","status":"completed","trials":[{"id":"2026-09-24T18-00-21-915Z-1f4155b6-9-1","index":1,"status":"passed","started_at":"2026-09-24T18:02:43.048Z","assessment_version":2,"method_id":"speedbench-fixed-payload-v19","method_sha256":"3279b43c011f3d2b12c530d007a9d8d093a4b41c99d29f93037ee0cddeb3bb04","finished_at":"2026-09-24T18:02:51.017Z","elapsed_ms":7968.7144,"verified_completion_ms":7968.7144,"completion_event_ms":7887.0154,"monotonic_clock":{"source":"windows_query_performance_counter","frequency_hz":10000000,"start_ticks":"6080889189390","end_ticks":"6080968876534"},"process_containment":{"method":"windows_job_object","verified":true,"normal_close_drain_ms":50,"descendants_required_termination":false},"exit_code":0,"output_bytes":2048,"output_sha256":"0b97a3fc6aa7c343bc799c5fe5ffaef85b77c99240a31c644f7e15f40de4deb5","prompt_sha256":"1feb8cfc6400bfb5a125580fc80e1feb11c91cfaed260644020d6a1f4d3970e3","cli_version":"Cursor SDK 1.0.31","requested_model":"grok-4.7-256k-high","observed_models":["grok-4.7-256k-high"],"model_verification":{"verified":true,"method":"explicit_sdk_pin+authenticated_catalog+sdk_result","server_attested":false},"provider_model_calls":null,"isolation_ok":true,"tools_used":false,"usage":{"input_tokens":5076,"output_tokens":299,"cache_read_input_tokens":2560,"cache_creation_input_tokens":0,"reasoning_tokens":42},"validation":{"ok":true,"reasons":[]},"retry_after_ms":0,"retry_blocked":false,"evidence_version":2,"inference_started":true,"raw_sha256":"37843fa9b2467b7324fafd1e835dfdc6153696cf8d480e71a9c58c6a5dc67755"}],"summary":{"result_ms":7968.7144,"selected_trial_id":"2026-09-24T18-00-21-915Z-1f4155b6-9-1","attempt_count":1,"valid_trials":1,"planned_trials":3,"mean_ms":null,"min_ms":null,"max_ms":null,"stddev_ms":null},"retry_decisions":[{"after_attempt":1,"at":"2026-09-24T18:02:51.112Z","retry":false,"delay_ms":0,"reason":"success"}],"stop_reason":"success"}],"validation":{"ok":true,"trial_count":10,"issues":[]}}]}