{ "schema_version": "1.0", "prepared_on": "2026-10-13", "timezone": "America/Vancouver", "status": "static evidence handoff; comparability review candidate", "preserved_input_sha256": { "MODEL-WEBSITE-HANDOFF.json": "40db7cc44c9d97a92e68466b877c3ddf20d951897ecf4cce215a19ea3d7056d2", "MODEL-WEBSITE-HANDOFF.md": "075ef9f7a55a5bca0f8d925cb23eb8434adb256db997643dc80f88ff0e9eef7b", "snapshots/deepswe-2026-10-12-2039.json": "1fc7559e95a94975faf6324ea92307e81145201a33c22d91f9a79e1e0477f70f" }, "latest_model_registry": [ { "key": "openai:luna:standard", "family": "openai:luna", "variant": "standard", "label": "GPT-6 Luna", "exact_api_id": "gpt-6-luna", "latest_version": "6", "latest_verified_on": "2026-10-12", "availability": "Documented API model; Work/Codex access depends on rollout/client/workspace.", "release_date": null, "verification_source": "https://developers.openai.com/api/docs/models", "board_label_match": null, "release_source": null }, { "key": "openai:sol:standard", "family": "openai:sol", "variant": "standard", "label": "GPT-6.1 Sol", "exact_api_id": "gpt-6.1-sol", "latest_version": "6.1", "latest_verified_on": "2026-10-12", "availability": "Documented API model; Work/Codex access depends on rollout/client/workspace.", "release_date": null, "verification_source": "https://developers.openai.com/api/docs/models", "board_label_match": null, "release_source": null }, { "key": "openai:astra:standard", "family": "openai:astra", "variant": "standard", "label": "GPT-6 Astra", "exact_api_id": "gpt-6-astra", "latest_version": "6", "latest_verified_on": "2026-10-12", "availability": "Documented API model.", "release_date": null, "verification_source": "https://developers.openai.com/api/docs/models", "board_label_match": "gpt-6-astra", "release_source": null }, { "key": "anthropic:haiku:standard", "family": "anthropic:haiku", "variant": "standard", "label": "Claude Haiku 5.5", "exact_api_id": "claude-haiku-5-5", "latest_version": "5.5", "latest_verified_on": "2026-10-12", "availability": "Released on Claude Platform and partners.", "release_date": "2026-10-07", "verification_source": "https://platform.claude.com/docs/en/models/overview", "board_label_match": null, "release_source": "https://www.anthropic.com/claude-haiku-5-5" }, { "key": "anthropic:sonnet:standard", "family": "anthropic:sonnet", "variant": "standard", "label": "Claude Sonnet 5.5", "exact_api_id": "claude-sonnet-5-5", "latest_version": "5.5", "latest_verified_on": "2026-10-12", "availability": "Released on Claude Platform and partners.", "release_date": "2026-09-28", "verification_source": "https://platform.claude.com/docs/en/models/overview", "board_label_match": null, "release_source": "https://www.anthropic.com/claude-sonnet-5-5" }, { "key": "anthropic:opus:standard", "family": "anthropic:opus", "variant": "standard", "label": "Claude Opus 5.5", "exact_api_id": "claude-opus-5-5", "latest_version": "5.5", "latest_verified_on": "2026-10-12", "availability": "Released on Claude Platform and partners.", "release_date": "2026-09-22", "verification_source": "https://platform.claude.com/docs/en/models/overview", "board_label_match": null, "release_source": "https://www.anthropic.com/claude-opus-5-5" }, { "key": "anthropic:fable:standard", "family": "anthropic:fable", "variant": "standard", "label": "Claude Fable 5.1", "exact_api_id": "claude-fable-5-1", "latest_version": "5.1", "latest_verified_on": "2026-10-12", "availability": "Current documented model for demanding reasoning.", "release_date": null, "verification_source": "https://platform.claude.com/docs/en/models/overview", "board_label_match": null, "release_source": null }, { "key": "google:flash-lite:standard", "family": "google:flash-lite", "variant": "standard", "label": "Gemini 3.5 Flash-Lite", "exact_api_id": "gemini-3.5-flash-lite", "latest_version": "3.5", "latest_verified_on": "2026-10-12", "availability": "Stable text-output API model; excludes TTS/image variants.", "release_date": null, "verification_source": "https://ai.google.dev/gemini-api/docs/models", "board_label_match": null, "release_source": null }, { "key": "google:flash:standard", "family": "google:flash", "variant": "standard", "label": "Gemini 3.8 Flash", "exact_api_id": "gemini-3.8-flash", "latest_version": "3.8", "latest_verified_on": "2026-10-12", "availability": "Stable text-output API model; excludes TTS/image/live variants.", "release_date": null, "verification_source": "https://ai.google.dev/gemini-api/docs/models", "board_label_match": "gemini-3.8-flash", "release_source": null }, { "key": "google:pro:standard", "family": "google:pro", "variant": "standard", "label": "Gemini 3.1 Pro preview", "exact_api_id": "gemini-3.1-pro-preview", "latest_version": "3.1", "latest_verified_on": "2026-10-12", "availability": "Preview. Board label gemini-3.1-pro does not establish this exact endpoint/snapshot.", "release_date": null, "verification_source": "https://ai.google.dev/gemini-api/docs/models", "board_label_match": null, "release_source": null }, { "key": "meta:muse-spark:standard", "family": "meta:muse-spark", "variant": "standard", "label": "Muse Spark 1.3", "exact_api_id": null, "latest_version": "1.3", "latest_verified_on": "2026-10-12", "availability": "Meta Model API and Muse Code; exact API ID unverified.", "release_date": "2026-09-02", "verification_source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "board_label_match": null, "release_source": "https://research.meta.ai/blog/introducing-muse-spark-1-3" }, { "key": "meta:muse-wonder:standard", "family": "meta:muse-wonder", "variant": "standard", "label": "Muse Wonder", "exact_api_id": null, "latest_version": "2026-10-12 launch", "latest_verified_on": "2026-10-12", "availability": "Meta Model API and Muse Code; exact API ID unverified.", "release_date": "2026-10-12", "verification_source": "https://research.meta.ai/blog/muse-wonder", "board_label_match": null, "release_source": "https://research.meta.ai/blog/muse-wonder" }, { "key": "meta:muse-wonder:full", "family": "meta:muse-wonder", "variant": "full", "label": "Muse Wonder Full", "exact_api_id": null, "latest_version": "2026-10-12 launch", "latest_verified_on": "2026-10-12", "availability": "Muse Code subscribers; public API access/ID unverified.", "release_date": "2026-10-12", "verification_source": "https://research.meta.ai/blog/muse-wonder", "board_label_match": null, "release_source": "https://research.meta.ai/blog/muse-wonder" }, { "key": "xai:grok:standard", "family": "xai:grok", "variant": "standard", "label": "Grok 4.7", "exact_api_id": "grok-4.7", "latest_version": "4.7", "latest_verified_on": "2026-10-12", "availability": "Documented API flagship; preserve xAI/Grok user label.", "release_date": "2026-09-21", "verification_source": "https://docs.x.ai/developers/models", "board_label_match": null, "release_source": "https://x.ai/news/grok-4-7" } ], "selected_views": [ { "id": "terminal-bench-4.0.0", "label": "Terminal-Bench4.0.0", "why": "Latest model coverage across all five requested providers; cost/token display and multiple effort settings.", "tasks": 66, "source": "https://www.tbench.ai/?version=4", "dataset_source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "model_only": false, "best_partial_cohorts": [ "Codex GPT-6.1 Sol max versus GPT-6 Astra max", "Claude Code Opus5.5 max versus Sonnet5.5 max" ], "caveat": "Different agents are system comparisons. Do not merge official Harbor scores with AA scores; 4.0 differs from2.0/2.1." }, { "id": "swe-atlas-qna", "label": "SWE-Atlas-QnA", "why": "Repository comprehension; AA current agent rows include latest Wonder/Grok/Sol/Claude, Scale owner adds Gemini.", "tasks": 124, "source": "https://labs.scale.com/leaderboard/sweatlas-qna", "dataset_source": "https://huggingface.co/datasets/ScaleAI/SWE-Atlas-QnA", "model_only": false, "best_partial_cohorts": [ "AA same-agent Claude Code latest variants at same named effort", "AA Codex Sol/Astra at max" ], "caveat": "Rubric Q&A, not patch coding. Separate Scale and AA cohorts; published pooled efficiency belongs to all303tasks." }, { "id": "swe-bench-pro-v2-hard51", "label": "SWE-Bench Pro V2 HARD-51", "why": "Locked protocol and small exact subset; useful same mini-swe-agent current cohort.", "tasks": 51, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "model_only": "partial candidate", "best_partial_cohorts": [ "Haiku5.5 high versus Gemini3.8 Flash high, mini-swe-agent", "Sonnet5.5 xhigh retained with different effort" ], "caveat": "Do not mix642Full/51HARD/731v1. Per-row protocol implementation/regrade/repeats require further verification." } ], "current_rows": [ { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 5.5", "current_model_key": "anthropic:opus:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 64.8, "ci_halfwidth_pp": 3.1, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 22, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "8.0B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$4.7k", "reported_cost_display": "$4.7k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 61.8, "ci_halfwidth_pp": 2.9, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 28, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "19.4B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$7.3k", "reported_cost_display": "$7.3k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 58.2, "ci_halfwidth_pp": 2.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 3, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.5B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$3.3k", "reported_cost_display": "$3.3k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 58.2, "ci_halfwidth_pp": 3.1, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 29, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.5B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$634.24", "reported_cost_display": "$634.24", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 57.9, "ci_halfwidth_pp": 3.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 1, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "2.7B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$6.2k", "reported_cost_display": "$6.2k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 57.9, "ci_halfwidth_pp": 2.7, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 3, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.2B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.4k", "reported_cost_display": "$2.4k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 57.9, "ci_halfwidth_pp": 3.0, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 3, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.2B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.3k", "reported_cost_display": "$2.3k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 57.9, "ci_halfwidth_pp": 3.4, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 1, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "2.3B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$4.9k", "reported_cost_display": "$4.9k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 54.5, "ci_halfwidth_pp": 3.4, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 1, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "2.2B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$4.0k", "reported_cost_display": "$4.0k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 54.2, "ci_halfwidth_pp": 2.7, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 3, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.1B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$1.9k", "reported_cost_display": "$1.9k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 53.9, "ci_halfwidth_pp": 3.4, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 1, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.6B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.8k", "reported_cost_display": "$2.8k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 50.6, "ci_halfwidth_pp": 2.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 3, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "889.8M", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$1.6k", "reported_cost_display": "$1.6k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 43.3, "ci_halfwidth_pp": 3.6, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 1, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "1.3B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.4k", "reported_cost_display": "$2.4k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Grok 4.7", "current_model_key": "xai:grok:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 37.6, "ci_halfwidth_pp": 3.5, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 21, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "5.5B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$3.68k (partial: 324/330)", "reported_cost_display": "$3.68k (partial: 324/330)", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": false, "cost_telemetry_attempts": 324, "cost_expected_attempts": 330, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Gemini 3.8 Flash", "current_model_key": "google:flash:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "mini-SWE-agent", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 19.1, "ci_halfwidth_pp": 3.4, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 2, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "17.2B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$1.8k", "reported_cost_display": "$1.8k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Luna", "current_model_key": "openai:luna:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 16.4, "ci_halfwidth_pp": 2.7, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 22, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "3.9B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$72.14", "reported_cost_display": "$72.14", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 14.5, "ci_halfwidth_pp": 2.9, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 2, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "9.0B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$0.00", "reported_cost_display": null, "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": false, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": "Harbor $0.00 is unavailable, not free; tbench.ai displays a dash." }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 66.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-0", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 67.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-0", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Opus 5.5", "current_model_key": "anthropic:opus:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 63.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-1", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Opus 5.5", "current_model_key": "anthropic:opus:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 66.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-1", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 58.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-2", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 62.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-2", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 58.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": true, "model_only_eligible": false, "fallback_note": "Explicit with fallback label", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-3", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 65.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": true, "model_only_eligible": false, "fallback_note": "Explicit with fallback label", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-3", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 42.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-5", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 56.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-5", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 27.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-7", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 45.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-7", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 25.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-9", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 39.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-9", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 24.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-10", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 52.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-10", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 30.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-11", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 34.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-11", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 13.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-12", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 46.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-12", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 12.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-13", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 42.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-13", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 6.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-14", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 33.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-14", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 55.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-15", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 61.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-15", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 56.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-16", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 62.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-16", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 52.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-17", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "medium", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 61.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-17", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 50.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-18", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 60.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-18", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 53.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-19", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 58.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-19", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 49.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-20", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "low", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 55.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-20", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6 Luna", "current_model_key": "openai:luna:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 15.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-25", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6 Luna", "current_model_key": "openai:luna:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 44.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-25", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Grok 4.7", "current_model_key": "xai:grok:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 33.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-27", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Grok 4.7", "current_model_key": "xai:grok:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 63.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-27", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Muse Wonder Full", "current_model_key": "meta:muse-wonder:full", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 59.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-29", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Muse Wonder Full", "current_model_key": "meta:muse-wonder:full", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 68.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-29", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Muse Wonder", "current_model_key": "meta:muse-wonder:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 47.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-30", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Muse Wonder", "current_model_key": "meta:muse-wonder:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 67.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-30", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 32.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-31", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "max", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 59.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-31", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 17.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-32", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Muse Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 55.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-32", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/sweatlas-qna", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 59.95, "ci_halfwidth_pp": 4.85, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "score_measure": "Task Resolve Rate: every rubric criterion passes", "task_split": "ScaleAI/SWE-Atlas-QnA test split", "dataset_source": "https://huggingface.co/datasets/ScaleAI/SWE-Atlas-QnA", "judge": "Claude Opus 4.5", "method_source": "https://labs.scale.com/leaderboard/sweatlas-qna", "attempts_per_task": null, "retry_policy": null, "step_limit_note": "Scale July28 note increased mini-swe-agent steps 250 to500; row-specific application not established", "pool_join_forbidden": true }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/sweatlas-qna", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 59.14, "ci_halfwidth_pp": 4.88, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "score_measure": "Task Resolve Rate: every rubric criterion passes", "task_split": "ScaleAI/SWE-Atlas-QnA test split", "dataset_source": "https://huggingface.co/datasets/ScaleAI/SWE-Atlas-QnA", "judge": "Claude Opus 4.5", "method_source": "https://labs.scale.com/leaderboard/sweatlas-qna", "attempts_per_task": null, "retry_policy": null, "step_limit_note": "Scale July28 note increased mini-swe-agent steps 250 to500; row-specific application not established", "pool_join_forbidden": true }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "model": "Gemini 3.8 Flash", "current_model_key": "google:flash:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": null, "agent_harness": "mini-SWE-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/sweatlas-qna", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 47.04, "ci_halfwidth_pp": 5.08, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "score_measure": "Task Resolve Rate: every rubric criterion passes", "task_split": "ScaleAI/SWE-Atlas-QnA test split", "dataset_source": "https://huggingface.co/datasets/ScaleAI/SWE-Atlas-QnA", "judge": "Claude Opus 4.5", "method_source": "https://labs.scale.com/leaderboard/sweatlas-qna", "attempts_per_task": null, "retry_policy": null, "step_limit_note": "Scale July28 note increased mini-swe-agent steps 250 to500; row-specific application not established", "pool_join_forbidden": true }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 92.2, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 90.2, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "xhigh", "agent_harness": "mini-swe-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 70.59, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Gemini 3.8 Flash", "current_model_key": "google:flash:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "mini-swe-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 58.8, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "version_status": "latest_in_accepted_registry", "effort": "high", "agent_harness": "mini-swe-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 52.94, "ci_halfwidth_pp": 13.01, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." } ], "history_rows": [ { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 53.9, "ci_halfwidth_pp": 3.2, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jul 24, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "6.9B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$6.1k", "reported_cost_display": "$6.1k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 51.8, "ci_halfwidth_pp": 3.4, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jul 24, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "6.5B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$6.0k", "reported_cost_display": "$6.0k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "high", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 50.3, "ci_halfwidth_pp": 3.7, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jul 24, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "5.5B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$4.7k", "reported_cost_display": "$4.7k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 49.4, "ci_halfwidth_pp": 3.2, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 22, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "5.0B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$1.5k", "reported_cost_display": "$1.5k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "medium", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 44.8, "ci_halfwidth_pp": 3.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jul 24, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "3.7B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$3.2k", "reported_cost_display": "$3.2k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Fable 5", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 44.5, "ci_halfwidth_pp": 3.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jun 9, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "3.8B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$7.3k", "reported_cost_display": "$7.3k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GLM-5.3", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 41.8, "ci_halfwidth_pp": 3.2, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Aug 14, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "8.7B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.7k", "reported_cost_display": "$2.7k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-5.6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 37.3, "ci_halfwidth_pp": 3.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jun 26, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "4.4B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.5k", "reported_cost_display": "$2.5k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GLM-5.3-Flash", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "none", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 35.8, "ci_halfwidth_pp": 3.5, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Aug 26, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "2.9B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.9k", "reported_cost_display": "$2.9k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "low", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 34.8, "ci_halfwidth_pp": 3.9, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jul 24, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "2.7B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.4k", "reported_cost_display": "$2.4k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Qwen3.8-Max-0902", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 27.0, "ci_halfwidth_pp": 3.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Sep 2, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "3.0B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$4.6k", "reported_cost_display": "$4.6k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Opus 4.8", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 23.6, "ci_halfwidth_pp": 3.6, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "May 28, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "6.4B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$6.5k", "reported_cost_display": "$6.5k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-5.6 Terra", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 21.5, "ci_halfwidth_pp": 3.3, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jun 26, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "5.7B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$1.7k", "reported_cost_display": "$1.7k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Grok 4.6", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "high", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 20.3, "ci_halfwidth_pp": 3.1, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Aug 12, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "4.0B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$3.6k", "reported_cost_display": "$3.6k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "GPT-5.6 Luna", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 17.3, "ci_halfwidth_pp": 2.8, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jun 26, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "11.6B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$346.67", "reported_cost_display": "$346.67", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Grok 4.5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "high", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 12.4, "ci_halfwidth_pp": 2.6, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jul 16, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "3.4B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$2.1k", "reported_cost_display": "$2.1k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Sonnet 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 12.4, "ci_halfwidth_pp": 3.1, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Jun 30, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "21.6B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$9.6k", "reported_cost_display": "$9.6k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "model": "Gemini 3.7 Flash", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "high", "agent_harness": "mini-SWE-agent", "harness_version": null, "serving_snapshot": null, "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 11.2, "ci_halfwidth_pp": 2.4, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "score_measure": "resolution rate", "ci_level": 0.95, "benchmark_version": "4.0.0", "task_split": "terminal-bench/terminal-bench rev.4", "model_release_date_display": "Aug 13, 2026", "declared_task_timeout_seconds": 28800, "attempts_per_task": null, "retry_policy": null, "method_source": "https://www.tbench.ai/news/terminal-bench-4-0", "reported_tokens_display": "11.1B", "token_scope": "board run aggregate display; token-category and attempt-coverage accounting unresolved", "raw_cost_display": "$1.3k", "reported_cost_display": "$1.3k", "cost_basis": "reported run cost; actual billing/rate date not established", "cost_pricing_date": null, "cost_complete": null, "cost_telemetry_attempts": null, "cost_expected_attempts": null, "zero_note": null }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 55.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-4", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 62.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-4", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Step 5 Preview", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "unspecified", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 33.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-6", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Step 5 Preview", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "unspecified", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 58.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-6", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Qwen3.8 Max", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "unspecified", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 17.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-8", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Qwen3.8 Max", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "unspecified", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 62.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-8", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 43.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-21", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 58.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-21", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-5.6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 37.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-22", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-5.6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 54.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-22", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "GPT-5.6 Luna", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 15.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-23", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "GPT-5.6 Luna", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 49.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-23", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "DeepSeek V4 Pro 0813", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 10.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-24", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "DeepSeek V4 Pro 0813", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 62.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-24", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "DeepSeek V4 Flash 0731", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 11.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-26", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "DeepSeek V4 Flash 0731", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 51.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-26", "benchmark_specific_cost_time_tokens": false }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "model": "Grok 4.6", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 66, "score_percent": 18.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "4.0", "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "program test suite", "declared_task_timeout_seconds": 28800, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-28", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "model": "Grok 4.6", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Grok Build", "harness_version": null, "serving_snapshot": null, "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 124, "score_percent": 58.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": null, "task_split": "AA methodology enumerated task IDs", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "attempts_per_task": 3, "score_measure": "pass@1: mean of 3 attempt scores/task, then mean across tasks", "fallback_enabled": null, "model_only_eligible": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "judge": "Claude Opus 4.5", "declared_task_timeout_seconds": null, "task_budget_applied_to_row_verified": false, "retry_policy": "provider safety errors re-run up to 10 times; model refusals score zero; fallback completions retained", "pooled_efficiency_ref": "aa-v1.5-pooled-28", "benchmark_specific_cost_time_tokens": false }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Opus 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 98.0, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Kimi-K3", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "mini-swe-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 88.2, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Sonnet 5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 88.2, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "GPT-5.6 Terra", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "xhigh", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 86.3, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "GLM-5.3", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "max", "agent_harness": "mini-swe-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 84.3, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "GPT-5.6 Sol", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Codex", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 82.4, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Inkling", "current_model_key": null, "bucket": "History", "version_status": "archival_family_latest_unverified", "effort": "xhigh", "agent_harness": "mini-swe-agent", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 56.9, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "model": "Haiku 4.5", "current_model_key": null, "bucket": "History", "version_status": "older_verified_family_version", "effort": "xhigh", "agent_harness": "Claude Code", "harness_version": null, "serving_snapshot": null, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "captured_on": "2026-10-13", "run_date": null, "benchmark_tasks": 51, "score_percent": 25.5, "ci_halfwidth_pp": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "model_only_comparability": "partial: identical harness label does not establish identical build/budget/settings", "numeric_origin": "source-reported aggregate; not independently re-run", "benchmark_version": "V2", "task_split": "HARD-51 (selected tab verified in browser)", "benchmark_update_date": "2026-09-22", "score_measure": "Resolve Rate", "dataset_source": "https://github.com/scaleapi/SWE-bench_Pro-os", "method_source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "declared_task_timeout_seconds": 3000, "attempts_per_task": null, "retry_policy": null, "grade_mode_applied_to_row": null, "protocol": "offline agent except model endpoint; web tools disabled; replay captured patch on pristine image", "comparability_review": "Candidate: HARD tab/51 IDs/harness verified; per-row build, repeats, timeout enforcement, regrade mode and grade artifacts not audited.", "legend_warning": "Page carries inherited 731-task prose and 50/250-turn legend. V2 README governs task protocol; do not silently apply old legend." } ], "missing_latest_placeholders": [ { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "current_model_key": "anthropic:haiku:standard", "model": "Claude Haiku 5.5", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://platform.claude.com/docs/en/models/overview" }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "current_model_key": "google:flash-lite:standard", "model": "Gemini 3.5 Flash-Lite", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "current_model_key": "google:pro:standard", "model": "Gemini 3.1 Pro preview", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "current_model_key": "meta:muse-wonder:standard", "model": "Muse Wonder", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/muse-wonder" }, { "view": "terminal-bench-4.0.0", "cohort": "official-harbor-4.0.0", "current_model_key": "meta:muse-wonder:full", "model": "Muse Wonder Full", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/muse-wonder" }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "current_model_key": "google:flash-lite:standard", "model": "Gemini 3.5 Flash-Lite", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "current_model_key": "google:flash:standard", "model": "Gemini 3.8 Flash", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "terminal-bench-4.0.0", "cohort": "artificial-analysis-v1.5", "current_model_key": "google:pro:standard", "model": "Gemini 3.1 Pro preview", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "current_model_key": "google:flash-lite:standard", "model": "Gemini 3.5 Flash-Lite", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "current_model_key": "google:flash:standard", "model": "Gemini 3.8 Flash", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "artificial-analysis-v1.5", "current_model_key": "google:pro:standard", "model": "Gemini 3.1 Pro preview", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "openai:luna:standard", "model": "GPT-6 Luna", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://developers.openai.com/api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "openai:sol:standard", "model": "GPT-6.1 Sol", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://developers.openai.com/api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "anthropic:haiku:standard", "model": "Claude Haiku 5.5", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://platform.claude.com/docs/en/models/overview" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "anthropic:sonnet:standard", "model": "Claude Sonnet 5.5", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://platform.claude.com/docs/en/models/overview" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "anthropic:opus:standard", "model": "Claude Opus 5.5", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://platform.claude.com/docs/en/models/overview" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "google:flash-lite:standard", "model": "Gemini 3.5 Flash-Lite", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "google:pro:standard", "model": "Gemini 3.1 Pro preview", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "meta:muse-spark:standard", "model": "Muse Spark 1.3", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "meta:muse-wonder:standard", "model": "Muse Wonder", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/muse-wonder" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "meta:muse-wonder:full", "model": "Muse Wonder Full", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/muse-wonder" }, { "view": "swe-atlas-qna", "cohort": "scale-owner-current-excerpt", "current_model_key": "xai:grok:standard", "model": "Grok 4.7", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://docs.x.ai/developers/models" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "openai:luna:standard", "model": "GPT-6 Luna", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://developers.openai.com/api/docs/models" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "openai:sol:standard", "model": "GPT-6.1 Sol", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://developers.openai.com/api/docs/models" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "anthropic:opus:standard", "model": "Claude Opus 5.5", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://platform.claude.com/docs/en/models/overview" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "google:flash-lite:standard", "model": "Gemini 3.5 Flash-Lite", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "google:pro:standard", "model": "Gemini 3.1 Pro preview", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://ai.google.dev/gemini-api/docs/models" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "meta:muse-spark:standard", "model": "Muse Spark 1.3", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "meta:muse-wonder:standard", "model": "Muse Wonder", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/muse-wonder" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "meta:muse-wonder:full", "model": "Muse Wonder Full", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://research.meta.ai/blog/muse-wonder" }, { "view": "swe-bench-pro-v2-hard51", "cohort": "scale-v2-hard-selected-ui", "current_model_key": "xai:grok:standard", "model": "Grok 4.7", "score_percent": null, "metrics": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null }, "status": "not established in inspected cohort; no older-version proxy", "source": "https://docs.x.ai/developers/models" } ], "pooled_efficiency": [ { "id": "aa-v1.5-pooled-0", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "27.7M", "reported_avg_agent_walltime_display": "1.5h", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "hours", "estimated_avg_api_cost_usd": 14.19, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-1", "model": "Opus 5.5", "current_model_key": "anthropic:opus:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "15.6M", "reported_avg_agent_walltime_display": "1.1h", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "hours", "estimated_avg_api_cost_usd": 13.04, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-2", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "xhigh", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "7.1M", "reported_avg_agent_walltime_display": "27.0m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 3.33, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-3", "model": "Fable 5.1", "current_model_key": "anthropic:fable:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "max", "fallback_enabled": true, "fallback_note": "Explicit with fallback label", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "5.7M", "reported_avg_agent_walltime_display": "34.8m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 12.39, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-4", "model": "Opus 5", "current_model_key": null, "bucket": "History", "agent_harness": "Claude Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "11.4M", "reported_avg_agent_walltime_display": "41.9m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 10.79, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-5", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "high", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "2.7M", "reported_avg_agent_walltime_display": "12.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 1.24, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-6", "model": "Step 5 Preview", "current_model_key": null, "bucket": "History", "agent_harness": "Claude Code", "effort": "unspecified", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "21.7M", "reported_avg_agent_walltime_display": "56.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 1.66, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-7", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "medium", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "1.3M", "reported_avg_agent_walltime_display": "8.5m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.62, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-8", "model": "Qwen3.8 Max", "current_model_key": null, "bucket": "History", "agent_harness": "Claude Code", "effort": "unspecified", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "8.7M", "reported_avg_agent_walltime_display": "1.1h", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "hours", "estimated_avg_api_cost_usd": 3.48, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-9", "model": "Sonnet 5.5", "current_model_key": "anthropic:sonnet:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "low", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "977.3K", "reported_avg_agent_walltime_display": "6.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.48, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-10", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "xhigh", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "7.1M", "reported_avg_agent_walltime_display": "25.9m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.61, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-11", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "13.8M", "reported_avg_agent_walltime_display": "1.5h", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "hours", "estimated_avg_api_cost_usd": 2.58, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-12", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "high", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "5.1M", "reported_avg_agent_walltime_display": "18.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.36, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-13", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "medium", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "3.5M", "reported_avg_agent_walltime_display": "14.0m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.23, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-14", "model": "Haiku 5.5", "current_model_key": "anthropic:haiku:standard", "bucket": "Current", "agent_harness": "Claude Code", "effort": "low", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "2.4M", "reported_avg_agent_walltime_display": "11.6m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.14, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-15", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "xhigh", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "3.2M", "reported_avg_agent_walltime_display": "15.5m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 1.04, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-16", "model": "GPT-6 Astra", "current_model_key": "openai:astra:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "3.3M", "reported_avg_agent_walltime_display": "29.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 7.47, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-17", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "medium", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "2.3M", "reported_avg_agent_walltime_display": "10.9m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.7, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-18", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "high", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "2.8M", "reported_avg_agent_walltime_display": "13.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.89, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-19", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "4M", "reported_avg_agent_walltime_display": "24.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 1.55, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-20", "model": "GPT-6.1 Sol", "current_model_key": "openai:sol:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "low", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "1.7M", "reported_avg_agent_walltime_display": "8.6m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.5, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-21", "model": "GPT-6 Sol", "current_model_key": null, "bucket": "History", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "9.8M", "reported_avg_agent_walltime_display": "22.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 2.99, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-22", "model": "GPT-5.6 Sol", "current_model_key": null, "bucket": "History", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "10.2M", "reported_avg_agent_walltime_display": "20.6m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 6.35, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-23", "model": "GPT-5.6 Luna", "current_model_key": null, "bucket": "History", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "14.9M", "reported_avg_agent_walltime_display": "23.5m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.44, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-24", "model": "DeepSeek V4 Pro 0813", "current_model_key": null, "bucket": "History", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "24.2M", "reported_avg_agent_walltime_display": "40.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.24, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-25", "model": "GPT-6 Luna", "current_model_key": "openai:luna:standard", "bucket": "Current", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "10.2M", "reported_avg_agent_walltime_display": "21.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.18, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-26", "model": "DeepSeek V4 Flash 0731", "current_model_key": null, "bucket": "History", "agent_harness": "Codex", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "15.4M", "reported_avg_agent_walltime_display": "18.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 0.09, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-27", "model": "Grok 4.7", "current_model_key": "xai:grok:standard", "bucket": "Current", "agent_harness": "Grok Build", "effort": "xhigh", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "14.3M", "reported_avg_agent_walltime_display": "39.2m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 8.82, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-28", "model": "Grok 4.6", "current_model_key": null, "bucket": "History", "agent_harness": "Grok Build", "effort": "xhigh", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "5.5M", "reported_avg_agent_walltime_display": "19.5m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 3.57, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-29", "model": "Muse Wonder Full", "current_model_key": "meta:muse-wonder:full", "bucket": "Current", "agent_harness": "Muse Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "15M", "reported_avg_agent_walltime_display": "28.8m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": null, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": "$0.00", "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-30", "model": "Muse Wonder", "current_model_key": "meta:muse-wonder:standard", "bucket": "Current", "agent_harness": "Muse Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "15M", "reported_avg_agent_walltime_display": "22.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 8.05, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-31", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "agent_harness": "Muse Code", "effort": "max", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "16.7M", "reported_avg_agent_walltime_display": "18.4m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 3.98, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } }, { "id": "aa-v1.5-pooled-32", "model": "Muse Spark 1.3", "current_model_key": "meta:muse-spark:standard", "bucket": "Current", "agent_harness": "Muse Code", "effort": "xhigh", "fallback_enabled": null, "fallback_note": "Not stated in row; absence is not proof of disabled fallback.", "source": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "method_source": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "scope": "pooled per-task-attempt averages across 303 tasks: DeepSWE v1.1 + Terminal-Bench4.0 + SWE-Atlas-QnA", "captured_on": "2026-10-13", "run_date": null, "metrics_coverage": null, "reported_avg_total_tokens_display": "16.2M", "reported_avg_agent_walltime_display": "18.3m", "time_boundary": "active agent process; excludes environment startup and verifier/judge overhead", "time_unit": "minutes", "estimated_avg_api_cost_usd": 3.47, "pricing_basis": "current provider token rates incl caching/cache writes where applicable; estimate, not actual bill", "pricing_effective_date": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "raw_missing_cost_display": null, "benchmark_specific_cost_time_tokens": false, "category_totals": { "input_tokens": null, "uncached_input_tokens": null, "cached_input_tokens": null, "cache_write_tokens": null, "output_tokens": null, "reasoning_tokens": null, "actual_total_tokens": null, "actual_billed_cost_usd": null, "total_attributable_cost_usd": null, "cost_per_success_usd": null, "duration_value": null, "duration_unit": null, "duration_measure": null, "agent_steps": null } } ], "vendor_claim_rows": [], "vendor_claim_policy": "Provider launch claims remain in earlier brief, outside evaluator cohorts.", "unselected_candidates": [ { "label": "Terminal-Bench2.0/2.1", "source": "https://www.tbench.ai/benchmarks", "reason": "Older versions; selected explicit4.0.0 instead. No numeric substitution." }, { "label": "LiveCodeBench", "source": "https://livecodebench.github.io/leaderboard.html", "reason": "Rendered default454problems, Aug1 2024\u2013May1 2025; newer scoped versions not established. Historical-only inspected coverage." }, { "label": "SWE-bench Verified Bash Only", "source": "https://www.swebench.com/", "reason": "500tasks/common mini-SWE-agent, but inspected model list historical and harness releases vary; latest scoped cohort not established." }, { "label": "SWE-Bench Pro v1", "source": "https://labs.scale.com/leaderboard/swe_bench_pro", "reason": "731-task older split. Selected explicitly versionedV2/HARD instead." } ], "task_level_evidence": [ { "view": "terminal-bench-4.0.0", "source": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "available": "Versioned task catalog inspected; no new individual trial telemetry extracted.", "measurements": [] }, { "view": "swe-atlas-qna", "source": "https://huggingface.co/datasets/ScaleAI/SWE-Atlas-QnA", "available": "124task IDs, base commits, rubrics, images; AA method enumerates tasks. No per-task consumption/duration extracted.", "measurements": [] }, { "view": "swe-bench-pro-v2-hard51", "source": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "available": "51exact IDs plus V2 task configs, grade/protocol tooling and advertised trajectories. Raw latest-model grades not audited.", "measurements": [] } ], "source_register": { "tb": "https://www.tbench.ai/?version=4", "harbor": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard", "tb_method": "https://www.tbench.ai/news/terminal-bench-4-0", "aa_cc": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex", "aa_gm": "https://artificialanalysis.ai/agents/coding-agents/comparisons/grok-build-vs-muse-code", "aa_method": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", "atlas": "https://labs.scale.com/leaderboard/sweatlas-qna", "atlas_data": "https://huggingface.co/datasets/ScaleAI/SWE-Atlas-QnA", "pro": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "pro_readme": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/main/v2/README.md", "pro_repo": "https://github.com/scaleapi/SWE-bench_Pro-os", "lcb": "https://livecodebench.github.io/leaderboard.html", "verified": "https://www.swebench.com/", "pro_v1": "https://labs.scale.com/leaderboard/swe_bench_pro", "muse": "https://research.meta.ai/blog/muse-wonder", "spark": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "grok": "https://docs.x.ai/developers/models" }, "rules": [ "Null displays blank, not0.", "Latest version onlyCurrent; History exact labels preserved.", "Agent/harness/source cohort is part of every row identity.", "No scores averaged across benchmarks.", "Pooled AA cost/time/tokens must never be copied into a single-benchmark row.", "Reported token/cost displays are not actual billed totals.", "Release dates are not run dates.", "Missing/zero-sentinel Muse costs are unknown, notfree.", "Fallback rows are mixed-model systems.", "No accepted DeepSWE data or previous website handoff changed." ] }