{
  "_ai_door": {
    "see": "https://miscsubjects.com/model-index",
    "note": "Every figure here is an observation with a source URL, a read date and an evidence class. Disagree by fetching the observation and reading its source, not by trusting this list."
  },
  "_self": {
    "what": "A living leaderboard of language models held as observations, not as a table of figures. Each row is one measurement of one model on one metric from one named source on one date.",
    "human_projection": "https://miscsubjects.com/model-index",
    "machine_projection": "https://miscsubjects.com/api/model-index",
    "raw_log": "https://miscsubjects.com/api/model-index/observations",
    "refresh_history": "https://miscsubjects.com/api/model-index/runs",
    "sources_registry": "https://miscsubjects.com/api/model-index/sources",
    "append_only": "A changed figure is a new observation; the old one is marked superseded and kept."
  },
  "evidence_classes": {
    "measured": "A number produced by running something and recording the result. The strongest class here.",
    "reported": "A named third party states it. Attribution required; not independently reproduced by this build.",
    "vendor_stated": "The seller states it about its own product. True as a price, interested as a claim.",
    "derived": "Computed by this build from two or more observations. The inputs are linked.",
    "promotional": "A price that is explicitly temporary. Carries an end date where one is published.",
    "unresolved": "Sources conflict and the evidence does not decide it. The conflict is shown, not averaged."
  },
  "metrics": {
    "price_in_usd_per_mtok": {
      "family": "price",
      "unit": "usd_per_mtok",
      "better": "lower",
      "what": "List price for one million input tokens at a named venue."
    },
    "price_out_usd_per_mtok": {
      "family": "price",
      "unit": "usd_per_mtok",
      "better": "lower",
      "what": "List price for one million output tokens at a named venue."
    },
    "cost_per_task_usd": {
      "family": "price",
      "unit": "usd",
      "better": "lower",
      "what": "Measured cost of one vendor's task suite. Not a production task price."
    },
    "price_cached_in_usd_per_mtok": {
      "family": "price",
      "unit": "usd_per_mtok",
      "better": "lower",
      "what": "Price of one million cache-hit input tokens. In an agent loop most input tokens are replayed context, so this is closer to the real input price than the list price is."
    },
    "swe_bench_verified": {
      "family": "capability",
      "unit": "fraction",
      "better": "higher",
      "what": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests."
    },
    "terminal_bench_2_1": {
      "family": "capability",
      "unit": "percent",
      "better": "higher",
      "what": "Agentic competence in a shell: running commands, reading output, recovering from failure."
    },
    "aa_intelligence_index": {
      "family": "capability",
      "unit": "index",
      "better": "higher",
      "what": "One vendor's composite of several evaluations. Index points are not a linear scale and are never converted to percentages."
    },
    "aa_ifbench": {
      "family": "obedience",
      "unit": "percent",
      "better": "higher",
      "what": "Ai2's IFBench: machine-checkable output constraints. Measures whether the model did what it was told, not whether it could."
    },
    "eqbench_creative_elo": {
      "family": "writing",
      "unit": "elo",
      "better": "higher",
      "what": "Elo from rated comparisons of prose. The only judged, rather than executed, metric here."
    },
    "openrouter_tokens_week": {
      "family": "popularity",
      "unit": "tokens_per_week",
      "better": "higher",
      "what": "Weekly token volume on one marketplace. Rewards cheap high-throughput work; free tiers inflate it."
    }
  },
  "last_refresh": {
    "run_id": "run_8fe360f3-547f-4937-89e7-26faa754eff3",
    "started_at": "2026-08-04T23:16:17-07:00",
    "finished_at": "2026-08-04T23:17:02-07:00",
    "sources_ok": 8,
    "sources_failed": 0,
    "observations": 820,
    "detail": "openrouter_models ok, 338 models | endpoints deepseek/deepseek-v4-flash ok, 21 venues | endpoints deepseek/deepseek-v4-pro ok, 18 venues | endpoints z-ai/glm-5.2 ok, 33 venues | endpoints moonshotai/kimi-k3 ok, 12 venues | endpoints minimax/minimax-m3 ok, 9 venues | endpoints qwen/qwen3.7-max ok, 1 venues | sources_registry verified 42/42"
  },
  "error": null,
  "count": 15,
  "observations": [
    {
      "model": {
        "key": "anthropic/claude-fable-5",
        "label": "Claude Fable 5",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.95,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_c0pz953yej"
    },
    {
      "model": {
        "key": "anthropic/claude-mythos-preview",
        "label": "Claude Mythos Preview",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.939,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_ndqi0scrsf"
    },
    {
      "model": {
        "key": "anthropic/claude-opus-4.8",
        "label": "Claude Opus 4.8",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.886,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_bhcqqksekw"
    },
    {
      "model": {
        "key": "anthropic/claude-opus-4.7",
        "label": "Claude Opus 4.7",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.876,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_w5142vuzlb"
    },
    {
      "model": {
        "key": "anthropic/claude-sonnet-5",
        "label": "Claude Sonnet 5",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.852,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_765enggf1s"
    },
    {
      "model": {
        "key": "deepseek/deepseek-v4-pro",
        "label": "DeepSeek V4 Pro Max",
        "maker": "DeepSeek",
        "weights": "open"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.806,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_l7amdp92s1"
    },
    {
      "model": {
        "key": "google/gemini-3.1-pro",
        "label": "Gemini 3.1 Pro",
        "maker": "Google",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.806,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_2971mkdnxw"
    },
    {
      "model": {
        "key": "minimax/minimax-m3",
        "label": "MiniMax M3",
        "maker": "MiniMax",
        "weights": "open"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.805,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_x9bwgdx1x1"
    },
    {
      "model": {
        "key": "qwen/qwen3.7-max",
        "label": "Qwen3.7 Max",
        "maker": "Alibaba",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.804,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_yhjuth3m5x"
    },
    {
      "model": {
        "key": "moonshotai/kimi-k2.6",
        "label": "Kimi K2.6",
        "maker": "Moonshot",
        "weights": "open"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.802,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_u8si2upj4n"
    },
    {
      "model": {
        "key": "openai/gpt-5.2",
        "label": "GPT-5.2",
        "maker": "OpenAI",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.8,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_bjyqijqfa4"
    },
    {
      "model": {
        "key": "deepseek/deepseek-v4-flash",
        "label": "DeepSeek V4 Flash Max",
        "maker": "DeepSeek",
        "weights": "open"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.79,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_jx4au9gee5"
    },
    {
      "model": {
        "key": "xiaomi/mimo-v2.5-pro",
        "label": "MiMo-V2.5-Pro",
        "maker": "Xiaomi",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.789,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_q4my1azbma"
    },
    {
      "model": {
        "key": "z-ai/glm-5",
        "label": "GLM-5",
        "maker": "Z.AI",
        "weights": "open"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.778,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_b2puih5sft"
    },
    {
      "model": {
        "key": "anthropic/claude-haiku-4.5",
        "label": "Claude Haiku 4.5",
        "maker": "Anthropic",
        "weights": "closed"
      },
      "metric": "swe_bench_verified",
      "metric_family": "capability",
      "what_it_measures": "Share of 500 human-validated real GitHub issues resolved, graded by running the repository tests.",
      "value": 0.733,
      "unit": "fraction",
      "venue": null,
      "evidence": {
        "class": "measured",
        "class_means": "A number produced by running something and recording the result. The strongest class here.",
        "method": "Real GitHub issues; a patch counts only if the repository tests pass.",
        "caveat": "Python repositories only, graded by the repository tests. It cannot see whether the model also changed files it was not asked to change."
      },
      "provenance": {
        "source_url": "https://llm-stats.com/benchmarks/swe-bench-verified",
        "title": "SWE-bench Verified leaderboard, 104 models",
        "publisher": "llm-stats",
        "source_type": "benchmark",
        "quote": null,
        "event_date": null,
        "read_at": "2026-08-04T00:00:00.000Z"
      },
      "observation_id": "mio_seed_305694lgvf"
    }
  ]
}