{
  "_ai_door": {
    "see": "https://miscsubjects.com/model-index",
    "note": "Every figure here is an observation with a source URL, a read date and an evidence class. Disagree by fetching the observation and reading its source, not by trusting this list."
  },
  "_self": {
    "what": "A living leaderboard of language models held as observations, not as a table of figures. Each row is one measurement of one model on one metric from one named source on one date.",
    "human_projection": "https://miscsubjects.com/model-index",
    "machine_projection": "https://miscsubjects.com/api/model-index",
    "raw_log": "https://miscsubjects.com/api/model-index/observations",
    "refresh_history": "https://miscsubjects.com/api/model-index/runs",
    "sources_registry": "https://miscsubjects.com/api/model-index/sources",
    "append_only": "A changed figure is a new observation; the old one is marked superseded and kept."
  },
  "evidence_classes": {
    "measured": "A number produced by running something and recording the result. The strongest class here.",
    "reported": "A named third party states it. Attribution required; not independently reproduced by this build.",
    "vendor_stated": "The seller states it about its own product. True as a price, interested as a claim.",
    "derived": "Computed by this build from two or more observations. The inputs are linked.",
    "promotional": "A price that is explicitly temporary. Carries an end date where one is published.",
    "unresolved": "Sources conflict and the evidence does not decide it. The conflict is shown, not averaged."
  },
  "metrics": {
    "price_in_usd_per_mtok": {
      "family": "price",
      "unit": "usd_per_mtok",
      "better": "lower",
      "what": "List price for one million input tokens at a named venue."
    },
    "price_out_usd_per_mtok": {
      "family": "price",
      "unit": "usd_per_mtok",
      "better": "lower",
      "what": "List price for one million output tokens at a named venue."
    },
    "cost_per_task_usd": {
      "family": "price",
      "unit": "usd",
      "better": "lower",
      "what": "Measured cost of one vendor's task suite. Not a production task price."
    },
    "price_cached_in_usd_per_mtok": {
      "family": "price",
      "unit": "usd_per_mtok",
      "better": "lower",
      "what": "Price of one million cache-hit input tokens. In an agent loop most input tokens are replayed context, so this is closer to the real input price than the list price is."
    },
    "swe_bench_verified": {
      "family": "capability",
      "unit": "fraction",
      "better": "higher",
      "what": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests."
    },
    "terminal_bench_2_1": {
      "family": "capability",
      "unit": "percent",
      "better": "higher",
      "what": "Agentic competence in a shell: running commands, reading output, recovering from failure."
    },
    "aa_intelligence_index": {
      "family": "capability",
      "unit": "index",
      "better": "higher",
      "what": "One vendor's composite of several evaluations. Index points are not a linear scale and are never converted to percentages."
    },
    "aa_ifbench": {
      "family": "obedience",
      "unit": "percent",
      "better": "higher",
      "what": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could."
    },
    "eqbench_creative_elo": {
      "family": "writing",
      "unit": "elo",
      "better": "higher",
      "what": "Elo from rated comparisons of prose. The only judged, rather than executed, metric here."
    },
    "openrouter_tokens_week": {
      "family": "popularity",
      "unit": "tokens_per_week",
      "better": "higher",
      "what": "Weekly token volume on one marketplace. Rewards cheap high-throughput work; free tiers inflate it."
    }
  },
  "last_refresh": {
    "run_id": "run_8fe360f3-547f-4937-89e7-26faa754eff3",
    "started_at": "2026-08-04T23:16:17-07:00",
    "finished_at": "2026-08-04T23:17:02-07:00",
    "sources_ok": 8,
    "sources_failed": 0,
    "observations": 820,
    "detail": "openrouter_models ok, 338 models | endpoints deepseek/deepseek-v4-flash ok, 21 venues | endpoints deepseek/deepseek-v4-pro ok, 18 venues | endpoints z-ai/glm-5.2 ok, 33 venues | endpoints moonshotai/kimi-k3 ok, 12 venues | endpoints minimax/minimax-m3 ok, 9 venues | endpoints qwen/qwen3.7-max ok, 1 venues | sources_registry verified 42/42"
  },
  "error": null,
  "count": 17,
  "observations": [
    {
      "model": {
        "key": "minimax/minimax-m3",
        "label": "MiniMax M3",
        "maker": "MiniMax",
        "weights": "open"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 82.9,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_minimax_ifb"
    },
    {
      "model": {
        "key": "nvidia/nemotron-3-ultra",
        "label": "Nemotron 3 Ultra",
        "maker": "Nvidia",
        "weights": "open"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 81.4,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_y7dzsfnt2q"
    },
    {
      "model": {
        "key": "x-ai/grok-4.3",
        "label": "Grok 4.3",
        "maker": "xAI",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 81.3,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_rj0bpm9a42"
    },
    {
      "model": {
        "key": "qwen/qwen3.7-max",
        "label": "Qwen3.7 Max",
        "maker": "Alibaba",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 80.5,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_y3hoes149e"
    },
    {
      "model": {
        "key": "xiaomi/mimo-v2.5-pro",
        "label": "MiMo-V2.5-Pro",
        "maker": "Xiaomi",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 79.9,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_yivm3q9lur"
    },
    {
      "model": {
        "key": "openai/gpt-5.2-codex",
        "label": "GPT-5.2-Codex",
        "maker": "OpenAI",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 77.6,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_vyqw9awtgh"
    },
    {
      "model": {
        "key": "google/gemini-3.1-pro",
        "label": "Gemini 3.1 Pro",
        "maker": "Google",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 77.1,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_dajncz5l56"
    },
    {
      "model": {
        "key": "deepseek/deepseek-v4-pro",
        "label": "DeepSeek V4 Pro (max)",
        "maker": "DeepSeek",
        "weights": "open"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 76.5,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_38ptjiuple"
    },
    {
      "model": {
        "key": "moonshotai/kimi-k2.6",
        "label": "Kimi K2.6",
        "maker": "Moonshot",
        "weights": "open"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 76,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_oeaz0sfwm9"
    },
    {
      "model": {
        "key": "openai/gpt-5.5",
        "label": "GPT-5.5",
        "maker": "OpenAI",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 75.9,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_yvtwxolf9j"
    },
    {
      "model": {
        "key": "z-ai/glm-5.2",
        "label": "GLM-5.2",
        "maker": "Z.AI",
        "weights": "open"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 73.3,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_2abosgbhzz"
    },
    {
      "model": {
        "key": "openai/gpt-5.6-sol",
        "label": "GPT-5.6 Sol",
        "maker": "OpenAI",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 72.7,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_dqt6zj493c"
    },
    {
      "model": {
        "key": "openai/gpt-5.6-terra",
        "label": "GPT-5.6 Terra",
        "maker": "OpenAI",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 71.2,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_wt6evpodpx"
    },
    {
      "model": {
        "key": "anthropic/claude-fable-5",
        "label": "Claude Fable 5",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 63.5,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_xin3gnd3lu"
    },
    {
      "model": {
        "key": "moonshotai/kimi-k2.7-code",
        "label": "Kimi K2.7 Code",
        "maker": "Moonshot",
        "weights": "open"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 63.1,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_ecgdlc360r"
    },
    {
      "model": {
        "key": "anthropic/claude-opus-4.8",
        "label": "Claude Opus 4.8",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 62.2,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_5fbt9466ud"
    },
    {
      "model": {
        "key": "anthropic/claude-sonnet-4.6",
        "label": "Claude Sonnet 4.6",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "aa_ifbench",
      "metric_family": "obedience",
      "what_it_measures": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could.",
      "value": 41.2,
      "unit": "percent",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Verifiable output constraints, graded by program. No judge model.",
        "caveat": "Machine-checkable constraints only. It does not measure whether the model could have done the task, only whether it did what it was told."
      },
      "provenance": {
        "source_url": "https://benchlm.ai/benchmarks/aaifbench",
        "title": "AA-IFBench leaderboard, 144 models",
        "publisher": "BenchLM mirror of Artificial Analysis / Ai2 IFBench",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_upre8rci9x"
    }
  ]
}