{
  "slug": "lunar-lander",
  "name": "Lunar Lander",
  "title": "One prompt. Four models. A referee that bites back.",
  "date": "2026-07-11",
  "status": "published",
  "thesis": "For the checkable part of a task the model still doesn't matter — but the moment the task gets open-ended, it matters enormously.",
  "verdict": "All four nailed the physics to ≈0% drift — and then the autopilot split them 20/20, 18/20, 5/20, 0/20. The two that tested their own work are the two that landed.",
  "prompt": "Build a single, self-contained HTML file (inline CSS and JavaScript, zero external dependencies, no CDN imports) implementing a playable 2D Lunar Lander game on a canvas, with a built-in autopilot and a self-test panel that grades the physics. The game: a side-view lander with rotation and a main thruster. Use exactly these constants: gravity 1.62 m/s², main-engine acceleration 4.5 m/s² along the lander's axis, rotation rate 90°/s, fuel for 10 seconds of continuous burn. The lander spawns near the top of the screen with random horizontal drift. Terrain is procedurally generated from a numeric seed (seed input field, default 42), deterministic for a given seed, with at least two flat landing pads. A landing is safe only if vertical speed ≤ 2.0 m/s, horizontal speed ≤ 1.0 m/s, and tilt ≤ 8° on a pad — anything else is a crash. Controls: left/right arrows rotate, up arrow thrusts, R restarts. Show a HUD with altitude, vertical speed, horizontal speed, fuel, and angle. The autopilot: an AUTO toggle that flies and lands the ship using only the same controls a human has. The self-test panel (always visible): one button runs the full suite headless at maximum speed and displays, persistently on screen: (1) PHYSICS DRIFT — simulate 10 seconds of thrust-free fall and report the percentage deviation from the analytic distance ½gt², which should be ≈ 0 for correct physics; (2) AUTOPILOT — run 20 episodes with seeds 1 through 20 and report landings succeeded out of 20, mean touchdown vertical speed, and mean fuel remaining.",
  "referee": "Two referees this time. Physics drift has an exact analytic answer (½gt²) — the checkable part. The autopilot must beat the model's own game across 20 seeded episodes — the open-ended part. Every number below is read from each app's own self-test panel.",
  "scoreLabel": "autopilot landings out of 20",
  "xPost": {
    "url": "https://x.com/0xBakeer/status/2075926760395333885",
    "text": "Round 2. Same 4 AI models, one harder prompt: a playable Lunar Lander with exact physics and an autopilot that must land its own game. 20 seeded episodes, self-graded.\n\nPhysics: all four ≈0% error. Again.\n\nAutopilot: 20/20, 18/20, 5/20, 0/20.\n\nThe difference wasn't the model.",
    "author": "Khaled Bakeer",
    "handle": "@0xBakeer",
    "date": "2026-07-11"
  },
  "models": [
    {
      "slug": "fable-5",
      "name": "Fable 5",
      "modelId": "claude-fable-5",
      "vendor": "Anthropic",
      "cli": "Claude Code",
      "confirmedVia": "session log in ~/.claude/projects",
      "accent": "#C07818",
      "bright": "#E8A33D",
      "score": 20,
      "scoreDisplay": "20/20",
      "lines": 658,
      "bytes": 25740,
      "costAsRun": 23.81,
      "costClean": null,
      "costDisplay": "~$23.81 as run",
      "costBasis": "Measured tokens × published pricing ($10/$50 per Mtok in/out, $1 cache-read, $12.5 cache-write): 220,781 output + 9.1M cache-read tokens. The bill is dominated by what the others skipped — it spent ~10 of its 19 minutes driving a real browser to test its own game before declaring done.",
      "tokens": {
        "input": 175,
        "output": 220781,
        "reasoning": null,
        "cacheRead": 9119287,
        "cacheWrite": 292308
      },
      "temperament": "The perfectionist again — wrote the game, then ran it end-to-end in a real browser, fixed what it found, and only then said done. 20/20.",
      "firstRenderRank": 2,
      "raceTime": "19:07",
      "raceSeconds": 1147,
      "selfTested": true,
      "drift": "8.77e-14 %",
      "autopilot": {
        "landed": 20,
        "total": 20,
        "meanVy": "1.14 m/s",
        "meanFuel": "2.89 s"
      }
    },
    {
      "slug": "grok-build",
      "name": "Grok Build",
      "modelId": "grok-build · badged “Grok 4.3” · 512K ctx",
      "vendor": "xAI",
      "cli": "Grok CLI v0.2.93",
      "confirmedVia": "~/.grok/sessions session store",
      "accent": "#1E88AE",
      "bright": "#4FC3E8",
      "score": 5,
      "scoreDisplay": "5/20",
      "lines": 1003,
      "bytes": 26738,
      "costAsRun": null,
      "costClean": null,
      "costDisplay": "subscription — no per-token price",
      "costBasis": "Ran on a subscription plan again; the CLI exposes no per-token price, so there is no honest dollar figure to put on a bar.",
      "tokens": {
        "input": null,
        "output": null,
        "reasoning": null,
        "cacheRead": null,
        "cacheWrite": null
      },
      "temperament": "The sprinter — done in 8:43, ten minutes before anyone else, perfect physics. Then its autopilot crashed 15 of 20 landings, touching down too hard on nearly empty tanks.",
      "firstRenderRank": 1,
      "raceTime": "8:43",
      "raceSeconds": 523,
      "selfTested": false,
      "drift": "≈0.000 %",
      "autopilot": {
        "landed": 5,
        "total": 20,
        "meanVy": "2.38 m/s",
        "meanFuel": "0.03 s"
      }
    },
    {
      "slug": "deepseek-v4-pro",
      "name": "DeepSeek V4 Pro",
      "modelId": "deepseek-v4-pro (max)",
      "vendor": "DeepSeek",
      "cli": "opencode",
      "confirmedVia": "opencode's SQLite session DB",
      "accent": "#2E9159",
      "bright": "#5BC98C",
      "score": 18,
      "scoreDisplay": "18/20",
      "lines": 917,
      "bytes": 30804,
      "costAsRun": 0.0825,
      "costClean": 0.0825,
      "costDisplay": "$0.0825 — metered, itemized",
      "costBasis": "The itemized bill again: 33,237 input + 25,534 output + 46,236 reasoning tokens, 1.5M cached — about eight cents for the second-best autopilot in the field.",
      "tokens": {
        "input": 33237,
        "output": 25534,
        "reasoning": 46236,
        "cacheRead": 1541760,
        "cacheWrite": null
      },
      "temperament": "The deliberator — slowest finisher, ran its own Node syntax checks before shipping, and landed 18/20 with the gentlest touchdowns in the field (0.06 m/s).",
      "firstRenderRank": 3,
      "raceTime": "20:31",
      "raceSeconds": 1231,
      "selfTested": true,
      "drift": "0.1667 %",
      "autopilot": {
        "landed": 18,
        "total": 20,
        "meanVy": "0.06 m/s",
        "meanFuel": "3.42 s"
      }
    },
    {
      "slug": "gemini-gemma-4",
      "name": "Gemini → Gemma 4",
      "modelId": "gemini-3.5-flash → gemma-4-31b-it (switched mid-run)",
      "vendor": "Google",
      "cli": "Gemini CLI",
      "confirmedVia": "the CLI's chat session log (both model IDs recorded)",
      "accent": "#B045C4",
      "bright": "#D774E8",
      "score": 0,
      "scoreDisplay": "0/20 (its own grader says FAIL)",
      "lines": 1422,
      "bytes": 52667,
      "costAsRun": null,
      "costClean": null,
      "costDisplay": "n/a — the CLI ran out of memory",
      "costBasis": "No honest bill exists, and the run never really finished. An API error forced a retry, the subscription quota ran out mid-task, and the run continued on gemma-4-31b-it until the CLI's own Node process died with a fatal V8 error — “Reached heap limit · Allocation failed · JavaScript heap out of memory” — after churning through ~16 GB of heap, crashing inside a JSON.stringify of its own accumulated context. The harness ran out of memory, not the model. What survived on disk is the shipped file, and its own grader reports 0/20.",
      "tokens": {
        "input": null,
        "output": null,
        "reasoning": null,
        "cacheRead": null,
        "cacheWrite": null
      },
      "temperament": "The DNF with the most style — biggest, most feature-heavy build again (1,422 lines, a retro flight-diagnostics terminal) with perfect physics. But the run never finished: after the quota-driven switch to gemma-4, the CLI's own Node process ran out of memory and crashed. The file that survived scores 0/20 on its own grader.",
      "firstRenderRank": 4,
      "raceTime": null,
      "raceSeconds": null,
      "selfTested": false,
      "drift": "0.000000 %",
      "autopilot": {
        "landed": 0,
        "total": 20,
        "meanVy": "—",
        "meanFuel": "—"
      }
    }
  ],
  "measurements": [
    {
      "slug": "physics",
      "name": "Physics",
      "unit": "free-fall drift vs ½gt²",
      "verdict": "The checkable part converged again — four implementations, all ≈0% drift. Test 01's thesis holds where it applied.",
      "spread": "8.77e-14 % → 0.17 %"
    },
    {
      "slug": "autopilot",
      "name": "Autopilot",
      "unit": "landings out of 20, self-graded",
      "verdict": "The open-ended part split the field wide open. The two that tested their own work are the two that landed.",
      "spread": "20/20 → 0/20"
    },
    {
      "slug": "cost",
      "name": "Cost",
      "unit": "USD for the same prompt",
      "verdict": "Eight cents bought 18/20. Twenty-four dollars bought 20/20 — and the ten minutes of self-testing that got it there.",
      "spread": "$0.0825 → ~$23.81"
    },
    {
      "slug": "size",
      "name": "Output size",
      "unit": "lines · kilobytes",
      "verdict": "The smallest build scored highest; the biggest scored zero. Size still tracks style, not quality.",
      "spread": "658 → 1,422 lines"
    },
    {
      "slug": "speed",
      "name": "Speed & temperament",
      "unit": "wall-clock, from the race recording",
      "verdict": "Fastest crashed most; slowest landed softest. This time you could pay in minutes, dollars, or landings — nobody avoided all three.",
      "spread": "8:43 → DNF"
    }
  ]
}