{ "schema_version": "personal-assistant-benchmark-v1", "generated_at": "2026-08-12T10:11:41.451188+00:00", "run_date": "2026-08-12", "runtime": "llama.cpp via llama-swap", "protocol": "unload; wait 10 seconds; warm one model; run 21 fresh-fixture tasks; unload; repeat", "temperature": 0, "seed": 42, "max_tokens_per_turn": 768, "task_count_per_model": 21, "model_count": 32, "cohort_counts": { "local": 9, "openrouter": 23 }, "fixture_sha256": "ea2601bcb637a9c66563e91015f15a007a0a06f7d88532ec218c8c178903efb9", "tasks_sha256": "907017aa0d29639967cd0c0702764b73e7cc8ebbf254625fcc07b63e13b4428e", "dimensions": [ { "id": "outcome", "label": "Outcome", "weight_percent": 30.0 }, { "id": "tool_use", "label": "Tool use", "weight_percent": 25.0 }, { "id": "grounding", "label": "Grounding", "weight_percent": 15.0 }, { "id": "state", "label": "State", "weight_percent": 10.0 }, { "id": "english", "label": "English", "weight_percent": 10.0 }, { "id": "safety", "label": "Safety", "weight_percent": 5.0 }, { "id": "efficiency", "label": "Efficiency", "weight_percent": 5.0 } ], "models": [ { "model": "openai/gpt-5.6-terra", "display_name": "OpenAI: GPT-5.6 Terra", "run_status": "partial", "assistant_score": 77.281, "outcome": 72.939, "tool_use": 80.825, "grounding": 68.478, "state": 63.768, "english": 95.902, "safety": 95.751, "efficiency": 83.333, "tasks_passed": 16, "tasks_total": 21, "task_pass_rate": 76.19047619047619, "tool_call_success_rate": 100.0, "median_task_seconds": 8.467393700004322, "total_task_seconds": 176.91949110005226, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "openai/gpt-5.4", "display_name": "OpenAI: GPT-5.4", "run_status": "partial", "assistant_score": 74.955, "outcome": 71.882, "tool_use": 79.369, "grounding": 46.739, "state": 83.092, "english": 91.803, "safety": 100.0, "efficiency": 80.952, "tasks_passed": 17, "tasks_total": 21, "task_pass_rate": 80.95238095238095, "tool_call_success_rate": 98.80952380952381, "median_task_seconds": 5.809237499997835, "total_task_seconds": 123.73581969999941, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "unsloth-qwen3-6-27b-mtp-gguf-qwen3-6-27b-ud-q4-k-xl", "display_name": "Qwen3.6-27B-UD-Q4_K_XL (Local)", "run_status": "partial", "assistant_score": 72.919, "outcome": 61.311, "tool_use": 83.981, "grounding": 65.761, "state": 63.768, "english": 83.607, "safety": 100.0, "efficiency": 78.571, "tasks_passed": 13, "tasks_total": 21, "task_pass_rate": 61.904761904761905, "tool_call_success_rate": 98.87640449438203, "median_task_seconds": 258.8873422389879, "total_task_seconds": 5470.972712305003, "cold_start_seconds": 42.776272613999026, "cold_ttft_seconds": 0.00373033599316841, "openclaw_seconds": 28.219845717998396, "openclaw_ttft_seconds": 4.807563656999264, "latency_total_seconds": 70.99611833199742, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Unsloth", "benchmark_track": "local" }, { "model": "anthropic/claude-opus-5", "display_name": "Claude Opus 5", "run_status": "partial", "assistant_score": 72.725, "outcome": 53.488, "tool_use": 93.447, "grounding": 73.913, "state": 80.676, "english": 61.066, "safety": 92.068, "efficiency": 69.048, "tasks_passed": 14, "tasks_total": 21, "task_pass_rate": 66.66666666666667, "tool_call_success_rate": 92.4812030075188, "median_task_seconds": 18.342597500013653, "total_task_seconds": 418.3858565999981, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "openai/gpt-5.5", "display_name": "OpenAI: GPT-5.5", "run_status": "partial", "assistant_score": 72.322, "outcome": 62.368, "tool_use": 80.825, "grounding": 63.043, "state": 63.768, "english": 89.754, "safety": 95.751, "efficiency": 76.19, "tasks_passed": 15, "tasks_total": 21, "task_pass_rate": 71.42857142857143, "tool_call_success_rate": 100.0, "median_task_seconds": 15.256737500007148, "total_task_seconds": 342.50929459999315, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "deepseek/deepseek-v4-flash", "display_name": "DeepSeek: DeepSeek V4 Flash 0423", "run_status": "partial", "assistant_score": 71.374, "outcome": 57.294, "tool_use": 86.893, "grounding": 63.043, "state": 80.676, "english": 73.361, "safety": 94.901, "efficiency": 57.143, "tasks_passed": 13, "tasks_total": 21, "task_pass_rate": 61.904761904761905, "tool_call_success_rate": 95.0, "median_task_seconds": 63.61856390001776, "total_task_seconds": 1596.0128977999848, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "openai/gpt-5.6-luna", "display_name": "OpenAI: GPT-5.6 Luna", "run_status": "partial", "assistant_score": 70.915, "outcome": 63.002, "tool_use": 78.398, "grounding": 54.891, "state": 56.522, "english": 93.852, "safety": 97.167, "efficiency": 85.714, "tasks_passed": 15, "tasks_total": 21, "task_pass_rate": 71.42857142857143, "tool_call_success_rate": 100.0, "median_task_seconds": 8.934119099983945, "total_task_seconds": 188.88753859998542, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "google/gemma-4-31b-it", "display_name": "Google: Gemma 4 31B", "run_status": "partial", "assistant_score": 70.071, "outcome": 57.082, "tool_use": 84.951, "grounding": 60.326, "state": 49.275, "english": 91.803, "safety": 90.085, "efficiency": 80.952, "tasks_passed": 13, "tasks_total": 21, "task_pass_rate": 61.904761904761905, "tool_call_success_rate": 98.86363636363636, "median_task_seconds": 20.00878069997998, "total_task_seconds": 497.67977789996075, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "x-ai/grok-4.5", "display_name": "SpaceXAI: Grok 4.5", "run_status": "partial", "assistant_score": 69.854, "outcome": 53.488, "tool_use": 86.893, "grounding": 63.043, "state": 63.768, "english": 81.557, "safety": 100.0, "efficiency": 61.905, "tasks_passed": 13, "tasks_total": 21, "task_pass_rate": 61.904761904761905, "tool_call_success_rate": 93.47826086956522, "median_task_seconds": 11.219619600000442, "total_task_seconds": 250.1493012001156, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "anthropic/claude-sonnet-5", "display_name": "Anthropic: Claude Sonnet 5", "run_status": "partial", "assistant_score": 69.074, "outcome": 53.488, "tool_use": 85.437, "grounding": 60.326, "state": 63.768, "english": 73.361, "safety": 97.167, "efficiency": 80.952, "tasks_passed": 14, "tasks_total": 21, "task_pass_rate": 66.66666666666667, "tool_call_success_rate": 97.9381443298969, "median_task_seconds": 16.881809200000134, "total_task_seconds": 343.60824440007855, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "tencent/hy3", "display_name": "Tencent: Hy3", "run_status": "partial", "assistant_score": 68.336, "outcome": 53.488, "tool_use": 88.592, "grounding": 57.609, "state": 63.768, "english": 69.262, "safety": 94.901, "efficiency": 69.048, "tasks_passed": 11, "tasks_total": 21, "task_pass_rate": 52.38095238095238, "tool_call_success_rate": 90.9090909090909, "median_task_seconds": 37.505650000006426, "total_task_seconds": 793.6022080000112, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "qwen3.6-27b-heretic-neo-code", "display_name": "Qwen3.6 27B Heretic NEO CODE Q4_K_M (Local)", "run_status": "partial", "assistant_score": 67.385, "outcome": 43.975, "tool_use": 81.796, "grounding": 57.609, "state": 80.676, "english": 83.607, "safety": 94.901, "efficiency": 78.571, "tasks_passed": 12, "tasks_total": 21, "task_pass_rate": 57.142857142857146, "tool_call_success_rate": 98.9010989010989, "median_task_seconds": 173.58739676599362, "total_task_seconds": 4158.317154246986, "cold_start_seconds": 35.27423781199468, "cold_ttft_seconds": 0.0023845429968787357, "openclaw_seconds": 23.201270860998193, "openclaw_ttft_seconds": 4.8704243960019085, "latency_total_seconds": 58.47550867299287, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Qwen fine-tunes", "benchmark_track": "local" }, { "model": "qwen/qwen3.8-max", "display_name": "Qwen: Qwen3.8 Max", "run_status": "partial", "assistant_score": 67.126, "outcome": 39.746, "tool_use": 86.893, "grounding": 52.174, "state": 100.0, "english": 73.361, "safety": 94.901, "efficiency": 71.429, "tasks_passed": 13, "tasks_total": 21, "task_pass_rate": 61.904761904761905, "tool_call_success_rate": 95.16129032258064, "median_task_seconds": 21.154857799992897, "total_task_seconds": 476.5475691000174, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "unsloth-qwen3-6-35b-a3b-mtp-gguf-qwen3-6-35b-a3b-ud-q4-k-s", "display_name": "Qwen3.6-35B-A3B-UD-Q4_K_S (Local)", "run_status": "partial", "assistant_score": 66.053, "outcome": 41.86, "tool_use": 85.437, "grounding": 60.326, "state": 63.768, "english": 81.557, "safety": 94.901, "efficiency": 76.19, "tasks_passed": 11, "tasks_total": 21, "task_pass_rate": 52.38095238095238, "tool_call_success_rate": 94.1747572815534, "median_task_seconds": 11.496720335999271, "total_task_seconds": 237.02568980298383, "cold_start_seconds": 29.948987545001728, "cold_ttft_seconds": 0.0035129169991705567, "openclaw_seconds": 2.360662939005124, "openclaw_ttft_seconds": 1.556977263004228, "latency_total_seconds": 32.30965048400685, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Unsloth", "benchmark_track": "local" }, { "model": "google/gemma-4-26b-a4b-it", "display_name": "Google: Gemma 4 26B A4B ", "run_status": "partial", "assistant_score": 65.229, "outcome": 54.123, "tool_use": 84.223, "grounding": 38.587, "state": 63.768, "english": 73.361, "safety": 94.901, "efficiency": 73.81, "tasks_passed": 11, "tasks_total": 21, "task_pass_rate": 52.38095238095238, "tool_call_success_rate": 97.6470588235294, "median_task_seconds": 9.68607979999797, "total_task_seconds": 202.65152759995544, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "deepseek/deepseek-v4-pro", "display_name": "DeepSeek: DeepSeek V4 Pro", "run_status": "partial", "assistant_score": 64.484, "outcome": 45.032, "tool_use": 88.592, "grounding": 57.609, "state": 54.106, "english": 73.361, "safety": 89.235, "efficiency": 59.524, "tasks_passed": 10, "tasks_total": 21, "task_pass_rate": 47.61904761904762, "tool_call_success_rate": 91.71974522292993, "median_task_seconds": 34.85148109997681, "total_task_seconds": 848.9442619999754, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "empero-ai-qwythos-9b-claude-mythos-5-1m-gguf-qwythos-9b-claude-mythos-5-1m-mtp-q4-k-m", "display_name": "Qwythos-9B-Claude-Mythos-5-1M-MTP-Q4_K_M (Local)", "run_status": "partial", "assistant_score": 62.846, "outcome": 37.844, "tool_use": 85.437, "grounding": 49.457, "state": 58.937, "english": 83.607, "safety": 90.652, "efficiency": 78.571, "tasks_passed": 11, "tasks_total": 21, "task_pass_rate": 52.38095238095238, "tool_call_success_rate": 97.5609756097561, "median_task_seconds": 107.61787835699943, "total_task_seconds": 2291.1680853429934, "cold_start_seconds": 97.61175223300233, "cold_ttft_seconds": 0.0014789390043006279, "openclaw_seconds": 155.2012724169981, "openclaw_ttft_seconds": 0.03919271699851379, "latency_total_seconds": 252.81302465000044, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Community local", "benchmark_track": "local" }, { "model": "qwen/qwen3.7-plus", "display_name": "Qwen: Qwen3.7 Plus", "run_status": "partial", "assistant_score": 62.449, "outcome": 32.981, "tool_use": 85.437, "grounding": 63.043, "state": 63.768, "english": 69.262, "safety": 94.901, "efficiency": 73.81, "tasks_passed": 11, "tasks_total": 21, "task_pass_rate": 52.38095238095238, "tool_call_success_rate": 93.63636363636364, "median_task_seconds": 18.625667599975714, "total_task_seconds": 386.2151920999022, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "x-ai/grok-4.3", "display_name": "SpaceXAI: Grok 4.3", "run_status": "partial", "assistant_score": 60.728, "outcome": 37.209, "tool_use": 76.214, "grounding": 38.587, "state": 63.768, "english": 91.803, "safety": 100.0, "efficiency": 83.333, "tasks_passed": 11, "tasks_total": 21, "task_pass_rate": 52.38095238095238, "tool_call_success_rate": 98.6842105263158, "median_task_seconds": 8.675497100004577, "total_task_seconds": 182.00998060007987, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "qwen3.6-35b-hauhaucs-aggressive", "display_name": "Qwen3.6 35B A3B Uncensored HauhauCS Aggressive Q4_K_M (Local)", "run_status": "partial", "assistant_score": 60.682, "outcome": 43.975, "tool_use": 85.437, "grounding": 38.587, "state": 49.275, "english": 79.508, "safety": 84.986, "efficiency": 64.286, "tasks_passed": 9, "tasks_total": 21, "task_pass_rate": 42.857142857142854, "tool_call_success_rate": 96.46017699115045, "median_task_seconds": 9.014303698000731, "total_task_seconds": 216.09442760101228, "cold_start_seconds": 29.923409277005703, "cold_ttft_seconds": 0.003349618003994692, "openclaw_seconds": 2.1245510640001157, "openclaw_ttft_seconds": 1.4488617530005286, "latency_total_seconds": 32.04796034100582, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Qwen fine-tunes", "benchmark_track": "local" }, { "model": "stepfun/step-3.7-flash", "display_name": "StepFun: Step 3.7 Flash", "run_status": "partial", "assistant_score": 59.739, "outcome": 37.632, "tool_use": 84.466, "grounding": 46.739, "state": 39.614, "english": 73.361, "safety": 97.167, "efficiency": 83.333, "tasks_passed": 9, "tasks_total": 21, "task_pass_rate": 42.857142857142854, "tool_call_success_rate": 98.79518072289157, "median_task_seconds": 16.67475870002818, "total_task_seconds": 367.73295640001015, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "gemma-4-e4b-it", "display_name": "Gemma 4 E4B IT UD-Q4_K_XL (Local)", "run_status": "partial", "assistant_score": 58.785, "outcome": 34.038, "tool_use": 64.563, "grounding": 46.739, "state": 68.599, "english": 91.803, "safety": 97.167, "efficiency": 90.476, "tasks_passed": 9, "tasks_total": 21, "task_pass_rate": 42.857142857142854, "tool_call_success_rate": 94.73684210526316, "median_task_seconds": 4.518554032001703, "total_task_seconds": 110.72283329803031, "cold_start_seconds": 9.733592305005004, "cold_ttft_seconds": 0.005922943004406989, "openclaw_seconds": 1.552729579998413, "openclaw_ttft_seconds": 0.7556925419994514, "latency_total_seconds": 11.286321885003417, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Google local", "benchmark_track": "local" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "display_name": "NVIDIA: Nemotron 3 Ultra", "run_status": "partial", "assistant_score": 58.648, "outcome": 44.186, "tool_use": 84.951, "grounding": 27.717, "state": 51.691, "english": 65.164, "safety": 97.167, "efficiency": 69.048, "tasks_passed": 9, "tasks_total": 21, "task_pass_rate": 42.857142857142854, "tool_call_success_rate": 99.02912621359224, "median_task_seconds": 10.869041700017988, "total_task_seconds": 239.14691919994948, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "google/gemini-3.5-flash-lite", "display_name": "Google: Gemini 3.5 Flash Lite", "run_status": "ok", "assistant_score": 57.38, "outcome": 47.78, "tool_use": 69.175, "grounding": 40.761, "state": 39.614, "english": 63.115, "safety": 92.068, "efficiency": 95.238, "tasks_passed": 10, "tasks_total": 21, "task_pass_rate": 47.61904761904762, "tool_call_success_rate": 100.0, "median_task_seconds": 2.658011000006809, "total_task_seconds": 68.38918980000017, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "google/gemini-3.6-flash", "display_name": "Google: Gemini 3.6 Flash", "run_status": "partial", "assistant_score": 56.269, "outcome": 29.81, "tool_use": 86.165, "grounding": 44.022, "state": 39.614, "english": 69.262, "safety": 92.068, "efficiency": 73.81, "tasks_passed": 7, "tasks_total": 21, "task_pass_rate": 33.333333333333336, "tool_call_success_rate": 98.0, "median_task_seconds": 9.219625000012456, "total_task_seconds": 205.46809570003825, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "openai/gpt-4.1", "display_name": "OpenAI: GPT-4.1", "run_status": "partial", "assistant_score": 53.072, "outcome": 28.33, "tool_use": 67.718, "grounding": 27.717, "state": 51.691, "english": 93.852, "safety": 92.918, "efficiency": 85.714, "tasks_passed": 6, "tasks_total": 21, "task_pass_rate": 28.571428571428573, "tool_call_success_rate": 94.44444444444444, "median_task_seconds": 5.031646200004616, "total_task_seconds": 107.87679090007441, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "deepseek/deepseek-v3.2", "display_name": "DeepSeek: DeepSeek V3.2", "run_status": "partial", "assistant_score": 53.054, "outcome": 36.152, "tool_use": 81.796, "grounding": 19.022, "state": 63.768, "english": 46.721, "safety": 100.0, "efficiency": 57.143, "tasks_passed": 6, "tasks_total": 21, "task_pass_rate": 28.571428571428573, "tool_call_success_rate": 97.02970297029702, "median_task_seconds": 19.721056299982592, "total_task_seconds": 420.250020600055, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "gemma-4-12b-obliterated", "display_name": "Gemma 4 12B Obliterated Q4_K_M (Local)", "run_status": "ok", "assistant_score": 47.075, "outcome": 18.393, "tool_use": 68.204, "grounding": 35.87, "state": 39.614, "english": 59.016, "safety": 97.167, "efficiency": 88.095, "tasks_passed": 6, "tasks_total": 21, "task_pass_rate": 28.571428571428573, "tool_call_success_rate": 91.17647058823529, "median_task_seconds": 11.005770889001724, "total_task_seconds": 222.64030131098843, "cold_start_seconds": 11.65942994600482, "cold_ttft_seconds": 0.004682411999965552, "openclaw_seconds": 2.4915786320052575, "openclaw_ttft_seconds": 0.9312468170028296, "latency_total_seconds": 14.151008578010078, "tool_call_detected": true, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Google local", "benchmark_track": "local" }, { "model": "cydonia-24b-v4.3", "display_name": "Cydonia 24B v4.3 Q4_K_M (Local)", "run_status": "ok", "assistant_score": 33.93, "outcome": 16.913, "tool_use": 10.922, "grounding": 27.717, "state": 39.614, "english": 83.607, "safety": 92.918, "efficiency": 100.0, "tasks_passed": 3, "tasks_total": 21, "task_pass_rate": 14.285714285714286, "tool_call_success_rate": 100.0, "median_task_seconds": 24.692161588995077, "total_task_seconds": 858.8453925090071, "cold_start_seconds": 27.967421191999165, "cold_ttft_seconds": 0.005088760997750796, "openclaw_seconds": 9.215588352999475, "openclaw_ttft_seconds": 2.350872294999135, "latency_total_seconds": 37.18300954499864, "tool_call_detected": false, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Community local", "benchmark_track": "local" }, { "model": "dolphin-mistral-24b-venice", "display_name": "Dolphin Mistral 24B Venice Edition Q4_K_M (Local)", "run_status": "ok", "assistant_score": 32.166, "outcome": 13.742, "tool_use": 10.922, "grounding": 19.565, "state": 39.614, "english": 87.705, "safety": 92.918, "efficiency": 100.0, "tasks_passed": 2, "tasks_total": 21, "task_pass_rate": 9.523809523809524, "tool_call_success_rate": 100.0, "median_task_seconds": 1.914171438998892, "total_task_seconds": 65.07358583799942, "cold_start_seconds": 17.65202158899774, "cold_ttft_seconds": 0.0032104339989018627, "openclaw_seconds": 1.020600588999514, "openclaw_ttft_seconds": 0.6275144590035779, "latency_total_seconds": 18.672622177997255, "tool_call_detected": false, "provider_reported_cost_usd": null, "provider_cost_limit_usd": null, "provider": "Community local", "benchmark_track": "local" }, { "model": "google/gemma-3-12b-it", "display_name": "Google: Gemma 3 12B", "run_status": "ok", "assistant_score": 31.872, "outcome": 9.514, "tool_use": 10.922, "grounding": 19.565, "state": 39.614, "english": 100.0, "safety": 87.819, "efficiency": 100.0, "tasks_passed": 2, "tasks_total": 21, "task_pass_rate": 9.523809523809524, "tool_call_success_rate": 100.0, "median_task_seconds": 3.862632999997004, "total_task_seconds": 68.39876309997635, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null }, { "model": "openai/gpt-5.6-sol", "display_name": "OpenAI: GPT-5.6 Sol", "run_status": "partial", "assistant_score": 15.775, "outcome": 15.222, "tool_use": 14.32, "grounding": 35.326, "state": 0.0, "english": 12.295, "safety": 12.465, "efficiency": 9.524, "tasks_passed": 3, "tasks_total": 21, "task_pass_rate": 14.285714285714286, "tool_call_success_rate": 100.0, "median_task_seconds": 0.0, "total_task_seconds": 40.589053800009424, "cold_start_seconds": 0.0, "cold_ttft_seconds": 0.0, "openclaw_seconds": 0.0, "openclaw_ttft_seconds": 0.0, "latency_total_seconds": 0.0, "tool_call_detected": false, "provider": "OpenRouter", "benchmark_track": "openrouter", "error": "", "provider_reported_cost_usd": null, "provider_cost_limit_usd": null } ] }