{ "status": "published", "generatedAt": "2026-08-09T07:03:49Z", "commit": "3e065a78", "suiteVersion": "1.0", "sourceUrl": "https://fuaran-ui.io/eval/README.md", "cohort": "since-20260730T003649Z", "summary": { "totalPrompts": 1235, "tasks": 57, "passed": 1015, "passRate": 0.8218623481781376, "passRateCi95": 0.05390585129480917, "parsePassRate": 0.8582995951417004, "providersEvaluated": 7, "reasoningShare": 0.5862801281812626, "costPerCorrectUsd": 0.08620864330049272, "costPerCorrectCachedUsd": 0.08432159527093605 }, "providers": [ { "name": "Google Gemini", "model": "gemini-3.6-flash", "tier": "headline", "budgetPick": true, "passed": 156, "total": 174, "passRate": 0.896551724137931, "passRateCi95": 0.06125654064999244, "parsePassRate": 0.9310344827586207, "meanOutputTokens": 1730.4770114942528, "costUsd": 10.8836751, "reasoningShare": 0.7330003334137309, "reasoningSplitSource": "native", "costPerCorrectUsd": 0.06976714807692308, "costPerCorrectCachedUsd": 0.06976714807692308 }, { "name": "OpenAI GPT", "model": "gpt-5.6-sol@low", "tier": "headline", "budgetPick": false, "passed": 148, "total": 171, "passRate": 0.8654970760233918, "passRateCi95": 0.07287929431105274, "parsePassRate": 0.9064327485380117, "meanOutputTokens": 1259.0116959064328, "costUsd": 9.380124, "reasoningShare": 0.19026700115466058, "reasoningSplitSource": "sdk", "costPerCorrectUsd": 0.0633792162162162, "costPerCorrectCachedUsd": 0.0633792162162162 }, { "name": "xAI Grok", "model": "grok-4.5", "tier": "headline", "budgetPick": true, "passed": 162, "total": 192, "passRate": 0.84375, "passRateCi95": 0.07129913627442923, "parsePassRate": 0.8645833333333334, "meanOutputTokens": 1582.5885416666667, "costUsd": 7.5064264, "reasoningShare": 0.4653417088963964, "reasoningSplitSource": "sdk", "costPerCorrectUsd": 0.046335965432098766, "costPerCorrectCachedUsd": 0.046335965432098766 }, { "name": "Claude (Opus)", "model": "claude-fable-5@low", "tier": "headline", "budgetPick": false, "passed": 143, "total": 171, "passRate": 0.8362573099415205, "passRateCi95": 0.08516981229021083, "parsePassRate": 0.8596491228070176, "meanOutputTokens": 1718.2280701754387, "costUsd": 23.011419, "reasoningShare": 0.16979506031211414, "reasoningSplitSource": "derived", "costPerCorrectUsd": 0.16091901398601405, "costPerCorrectCachedUsd": 0.15360823076923083 }, { "name": "Google Gemini", "model": "gemini-3.1-pro-preview", "tier": "budget", "budgetPick": false, "passed": 140, "total": 171, "passRate": 0.8187134502923976, "passRateCi95": 0.08508713651053411, "parsePassRate": 0.8421052631578947, "meanOutputTokens": 1587.7251461988303, "costUsd": 21.8689392, "reasoningShare": 0.827090572363125, "reasoningSplitSource": "native", "costPerCorrectUsd": 0.15620670857142854, "costPerCorrectCachedUsd": 0.15620670857142854 }, { "name": "Claude (Opus)", "model": "claude-opus-4-8@low", "tier": "budget", "budgetPick": true, "passed": 145, "total": 180, "passRate": 0.8055555555555556, "passRateCi95": 0.07958804366465794, "parsePassRate": 0.8833333333333333, "meanOutputTokens": 1626.5055555555555, "costUsd": 11.05585175, "reasoningShare": 0.08672310744544127, "reasoningSplitSource": "derived", "costPerCorrectUsd": 0.07624725344827585, "costPerCorrectCachedUsd": 0.07024786206896551 }, { "name": "OpenAI GPT", "model": "gpt-5.6-terra@low", "tier": "budget", "budgetPick": true, "passed": 121, "total": 176, "passRate": 0.6875, "passRateCi95": 0.09236045985947963, "parsePassRate": 0.7215909090909091, "meanOutputTokens": 818.7556818181819, "costUsd": 3.7953375, "reasoningShare": 0.2737284035239804, "reasoningSplitSource": "sdk", "costPerCorrectUsd": 0.03136642561983471, "costPerCorrectCachedUsd": 0.03136642561983471 } ], "compactness": [ { "complexity": "all", "baseline": "JSX \u002B shadcn/ui", "baselineId": "jsx_shadcn", "fuaranTokens": 1464.123836336336, "baselineTokens": 2047.6405405405412, "ratio": 0.7150297170566045, "fuaranBilledTokens": 3408.637975475476, "baselineBilledTokens": 2893.319744744746, "billedRatio": 1.178106215763648, "fuaranColdCallUsd": 0.06648314943881382, "baselineColdCallUsd": 0.046039381876876874, "coldCallBillRatio": 1.4440495664474757, "fuaranCachedTurnUsd": 0.06514184977352351, "baselineCachedTurnUsd": 0.04595064539039039, "cachedTurnBillRatio": 1.417648201022799, "pairs": 222, "tasks": 57 }, { "complexity": "all", "baseline": "Feliz (bare)", "baselineId": "feliz_bare", "fuaranTokens": 1366.3665146489775, "baselineTokens": 2021.809203980099, "ratio": 0.67581377706619, "fuaranBilledTokens": 3330.749571586513, "baselineBilledTokens": 3269.9256218905484, "billedRatio": 1.0186010193286288, "fuaranColdCallUsd": 0.06492505672125483, "baselineColdCallUsd": 0.05229804233001658, "coldCallBillRatio": 1.241443347182251, "fuaranCachedTurnUsd": 0.06344362127003868, "baselineCachedTurnUsd": 0.05225853611111111, "cachedTurnBillRatio": 1.2140336486874805, "pairs": 201, "tasks": 54 }, { "complexity": "all", "baseline": "Dash (Python)", "baselineId": "dash_py", "fuaranTokens": 1800.1175105485238, "baselineTokens": 2806.6561181434604, "ratio": 0.6413744451668917, "fuaranBilledTokens": 3733.8468354430383, "baselineBilledTokens": 4191.801687763715, "billedRatio": 0.8907498764415569, "fuaranColdCallUsd": 0.07361931185654008, "baselineColdCallUsd": 0.05997801751054853, "coldCallBillRatio": 1.2274382334093055, "fuaranCachedTurnUsd": 0.07116958137130802, "baselineCachedTurnUsd": 0.05981352383966245, "cachedTurnBillRatio": 1.1898576910814833, "pairs": 79, "tasks": 20 }, { "complexity": "trivial", "baseline": "JSX \u002B shadcn/ui", "baselineId": "jsx_shadcn", "fuaranTokens": 289.61979166666674, "baselineTokens": 879.071875, "ratio": 0.32946087789086276, "fuaranBilledTokens": 1026.0447916666667, "baselineBilledTokens": 1460.63125, "billedRatio": 0.7024666846383485, "fuaranColdCallUsd": 0.029839614583333337, "baselineColdCallUsd": 0.021047369791666666, "coldCallBillRatio": 1.4177360344164145, "fuaranCachedTurnUsd": 0.029197920572916668, "baselineCachedTurnUsd": 0.021047369791666666, "cachedTurnBillRatio": 1.3872479488851415, "pairs": 32, "tasks": 8 }, { "complexity": "trivial", "baseline": "Feliz (bare)", "baselineId": "feliz_bare", "fuaranTokens": 289.61979166666674, "baselineTokens": 647.4677083333332, "ratio": 0.4473115615482756, "fuaranBilledTokens": 1026.0447916666667, "baselineBilledTokens": 1261.1619791666667, "billedRatio": 0.8135709834391317, "fuaranColdCallUsd": 0.029839614583333337, "baselineColdCallUsd": 0.019517895833333333, "coldCallBillRatio": 1.5288335811472167, "fuaranCachedTurnUsd": 0.029197920572916668, "baselineCachedTurnUsd": 0.019517895833333333, "cachedTurnBillRatio": 1.4959563685677355, "pairs": 32, "tasks": 8 }, { "complexity": "trivial", "baseline": "Dash (Python)", "baselineId": "dash_py", "fuaranTokens": 371.96666666666664, "baselineTokens": 1541.1458333333335, "ratio": 0.24135721527543086, "fuaranBilledTokens": 1046.0041666666666, "baselineBilledTokens": 2226.5, "billedRatio": 0.4697975147840407, "fuaranColdCallUsd": 0.029372919166666667, "baselineColdCallUsd": 0.031276986458333336, "coldCallBillRatio": 0.9391224185167828, "fuaranCachedTurnUsd": 0.029372919166666667, "baselineCachedTurnUsd": 0.031276986458333336, "cachedTurnBillRatio": 0.9391224185167828, "pairs": 8, "tasks": 2 }, { "complexity": "medium", "baseline": "JSX \u002B shadcn/ui", "baselineId": "jsx_shadcn", "fuaranTokens": 1048.1814814814813, "baselineTokens": 1522.7377104377101, "ratio": 0.6883532694413814, "fuaranBilledTokens": 3044.158080808082, "baselineBilledTokens": 2250.690235690236, "billedRatio": 1.3525442251161264, "fuaranColdCallUsd": 0.05760473558922559, "baselineColdCallUsd": 0.035700897727272723, "coldCallBillRatio": 1.613537453015363, "fuaranCachedTurnUsd": 0.05716473474747476, "baselineCachedTurnUsd": 0.035700897727272723, "cachedTurnBillRatio": 1.6012128093856117, "pairs": 99, "tasks": 26 }, { "complexity": "medium", "baseline": "Feliz (bare)", "baselineId": "feliz_bare", "fuaranTokens": 1049.958422939068, "baselineTokens": 1613.1709677419353, "ratio": 0.6508661784366018, "fuaranBilledTokens": 3105.2569892473134, "baselineBilledTokens": 2817.6951612903217, "billedRatio": 1.1020556914415494, "fuaranColdCallUsd": 0.058123608046595006, "baselineColdCallUsd": 0.04446793969534052, "coldCallBillRatio": 1.307090196775754, "fuaranCachedTurnUsd": 0.05765522005376346, "baselineCachedTurnUsd": 0.04443947822580647, "cachedTurnBillRatio": 1.2973874211754914, "pairs": 93, "tasks": 25 }, { "complexity": "medium", "baseline": "Dash (Python)", "baselineId": "dash_py", "fuaranTokens": 984.9938271604941, "baselineTokens": 2089.777777777778, "ratio": 0.47133902830946656, "fuaranBilledTokens": 2807.1604938271603, "baselineBilledTokens": 3429.3518518518513, "billedRatio": 0.8185688185688187, "fuaranColdCallUsd": 0.055973231172839505, "baselineColdCallUsd": 0.04587168209876544, "coldCallBillRatio": 1.2202131818999926, "fuaranCachedTurnUsd": 0.05435989475308642, "baselineCachedTurnUsd": 0.04587168209876544, "cachedTurnBillRatio": 1.1850425418463002, "pairs": 27, "tasks": 7 }, { "complexity": "compound", "baseline": "JSX \u002B shadcn/ui", "baselineId": "jsx_shadcn", "fuaranTokens": 2329.644963369964, "baselineTokens": 3029.6139194139205, "ratio": 0.7689577039640201, "fuaranBilledTokens": 4642.9950244200245, "baselineBilledTokens": 4096.246703296704, "billedRatio": 1.1334754375714946, "fuaranColdCallUsd": 0.08902772181776558, "baselineColdCallUsd": 0.06607514360805858, "coldCallBillRatio": 1.3473708410814211, "fuaranCachedTurnUsd": 0.08645987309218563, "baselineCachedTurnUsd": 0.06585866558608056, "cachedTurnBillRatio": 1.3128093671922074, "pairs": 91, "tasks": 23 }, { "complexity": "compound", "baseline": "Feliz (bare)", "baselineId": "feliz_bare", "fuaranTokens": 2206.917141812866, "baselineTokens": 3100.5234649122813, "ratio": 0.7117885630564976, "fuaranBilledTokens": 4577.083296783625, "baselineBilledTokens": 4669.108114035087, "billedRatio": 0.9802907075604352, "fuaranColdCallUsd": 0.088020699815424, "baselineColdCallUsd": 0.07568178223684212, "coldCallBillRatio": 1.1630368262201847, "fuaranCachedTurnUsd": 0.08494603884137429, "baselineCachedTurnUsd": 0.07561212653508774, "cachedTurnBillRatio": 1.1234446474925575, "pairs": 76, "tasks": 21 }, { "complexity": "compound", "baseline": "Dash (Python)", "baselineId": "dash_py", "fuaranTokens": 2559.9708333333338, "baselineTokens": 3476.6515151515136, "ratio": 0.7363323077325366, "fuaranBilledTokens": 4791.193939393939, "baselineBilledTokens": 5016.996212121211, "billedRatio": 0.95499253673309, "fuaranColdCallUsd": 0.09249238731060606, "baselineColdCallUsd": 0.07385254715909091, "coldCallBillRatio": 1.2523926508771834, "fuaranCachedTurnUsd": 0.08908400946969695, "baselineCachedTurnUsd": 0.07355720625, "cachedTurnBillRatio": 1.2110847327034928, "pairs": 44, "tasks": 11 } ], "costPerCorrect": [ { "condition": "Fuaran", "conditionId": "fuaran", "runs": 1235, "passRate": 0.8218623481781376, "meanColdCallUsd": 0.07085163801619442, "meanCachedTurnUsd": 0.06930074429149805, "coldUsdPerCorrect": 0.08620864330049272, "cachedUsdPerCorrect": 0.08432159527093605 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "Claude (Opus)", "providerModelId": "claude-fable-5@low", "runs": 171, "passRate": 0.8362573099415205, "meanColdCallUsd": 0.134569701754386, "meanCachedTurnUsd": 0.12845600584795328, "coldUsdPerCorrect": 0.16091901398601405, "cachedUsdPerCorrect": 0.15360823076923083 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "Claude (Opus)", "providerModelId": "claude-opus-4-8@low", "runs": 180, "passRate": 0.8055555555555556, "meanColdCallUsd": 0.0614213986111111, "meanCachedTurnUsd": 0.056588555555555545, "coldUsdPerCorrect": 0.07624725344827585, "cachedUsdPerCorrect": 0.07024786206896551 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "Google Gemini", "providerModelId": "gemini-3.1-pro-preview", "runs": 171, "passRate": 0.8187134502923976, "meanColdCallUsd": 0.1278885333333333, "meanCachedTurnUsd": 0.1278885333333333, "coldUsdPerCorrect": 0.15620670857142854, "cachedUsdPerCorrect": 0.15620670857142854 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "Google Gemini", "providerModelId": "gemini-3.6-flash", "runs": 174, "passRate": 0.896551724137931, "meanColdCallUsd": 0.06254985689655174, "meanCachedTurnUsd": 0.06254985689655174, "coldUsdPerCorrect": 0.0697671480769231, "cachedUsdPerCorrect": 0.0697671480769231 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-sol@low", "runs": 171, "passRate": 0.8654970760233918, "meanColdCallUsd": 0.05485452631578946, "meanCachedTurnUsd": 0.05485452631578946, "coldUsdPerCorrect": 0.0633792162162162, "cachedUsdPerCorrect": 0.0633792162162162 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-terra@low", "runs": 176, "passRate": 0.6875, "meanColdCallUsd": 0.021564417613636363, "meanCachedTurnUsd": 0.021564417613636363, "coldUsdPerCorrect": 0.03136642561983471, "cachedUsdPerCorrect": 0.03136642561983471 }, { "condition": "Fuaran", "conditionId": "fuaran", "provider": "xAI Grok", "providerModelId": "grok-4.5", "runs": 192, "passRate": 0.84375, "meanColdCallUsd": 0.039095970833333334, "meanCachedTurnUsd": 0.039095970833333334, "coldUsdPerCorrect": 0.046335965432098766, "cachedUsdPerCorrect": 0.046335965432098766 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "runs": 1200, "passRate": 0.9841666666666666, "meanColdCallUsd": 0.049855587666666715, "meanCachedTurnUsd": 0.049757090166666705, "coldUsdPerCorrect": 0.050657667400508095, "cachedUsdPerCorrect": 0.05055758526672316 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "Claude (Opus)", "providerModelId": "claude-fable-5@low", "runs": 171, "passRate": 0.9883040935672515, "meanColdCallUsd": 0.09343629239766085, "meanCachedTurnUsd": 0.09312908771929827, "coldUsdPerCorrect": 0.09454204733727814, "cachedUsdPerCorrect": 0.09423120710059174 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "Claude (Opus)", "providerModelId": "claude-opus-4-8@low", "runs": 171, "passRate": 0.9824561403508771, "meanColdCallUsd": 0.0488384619883041, "meanCachedTurnUsd": 0.04845445614035088, "coldUsdPerCorrect": 0.04971057738095239, "cachedUsdPerCorrect": 0.049319714285714286 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "Google Gemini", "providerModelId": "gemini-3.1-pro-preview", "runs": 171, "passRate": 0.9707602339181286, "meanColdCallUsd": 0.05924340350877194, "meanCachedTurnUsd": 0.05924340350877194, "coldUsdPerCorrect": 0.06102784337349399, "cachedUsdPerCorrect": 0.06102784337349399 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "Google Gemini", "providerModelId": "gemini-3.6-flash", "runs": 165, "passRate": 0.9939393939393939, "meanColdCallUsd": 0.06243571818181819, "meanCachedTurnUsd": 0.06243571818181819, "coldUsdPerCorrect": 0.06281642378048781, "cachedUsdPerCorrect": 0.06281642378048781 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-sol@low", "runs": 171, "passRate": 0.9766081871345029, "meanColdCallUsd": 0.050225868421052616, "meanCachedTurnUsd": 0.050225868421052616, "coldUsdPerCorrect": 0.05142888323353292, "cachedUsdPerCorrect": 0.05142888323353292 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-terra@low", "runs": 168, "passRate": 0.9821428571428571, "meanColdCallUsd": 0.023596214285714283, "meanCachedTurnUsd": 0.023596214285714283, "coldUsdPerCorrect": 0.024025236363636363, "cachedUsdPerCorrect": 0.024025236363636363 }, { "condition": "JSX \u002B shadcn/ui", "conditionId": "jsx_shadcn", "provider": "xAI Grok", "providerModelId": "grok-4.5", "runs": 183, "passRate": 0.994535519125683, "meanColdCallUsd": 0.013729066666666663, "meanCachedTurnUsd": 0.013729066666666663, "coldUsdPerCorrect": 0.013804501098901094, "cachedUsdPerCorrect": 0.013804501098901094 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "runs": 1200, "passRate": 0.7983333333333333, "meanColdCallUsd": 0.05619536487500001, "meanCachedTurnUsd": 0.056102722791666684, "coldUsdPerCorrect": 0.07039085370563675, "cachedUsdPerCorrect": 0.07027480934237998 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "Claude (Opus)", "providerModelId": "claude-fable-5@low", "runs": 171, "passRate": 0.8421052631578947, "meanColdCallUsd": 0.09900073099415202, "meanCachedTurnUsd": 0.09881498245614034, "coldUsdPerCorrect": 0.11756336805555553, "cachedUsdPerCorrect": 0.11734279166666667 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "Claude (Opus)", "providerModelId": "claude-opus-4-8@low", "runs": 171, "passRate": 0.7777777777777778, "meanColdCallUsd": 0.04904159941520465, "meanCachedTurnUsd": 0.04857722807017541, "coldUsdPerCorrect": 0.06305348496240597, "cachedUsdPerCorrect": 0.062456436090225526 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "Google Gemini", "providerModelId": "gemini-3.1-pro-preview", "runs": 171, "passRate": 0.847953216374269, "meanColdCallUsd": 0.0874244912280702, "meanCachedTurnUsd": 0.0874244912280702, "coldUsdPerCorrect": 0.10310060689655175, "cachedUsdPerCorrect": 0.10310060689655175 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "Google Gemini", "providerModelId": "gemini-3.6-flash", "runs": 165, "passRate": 0.896969696969697, "meanColdCallUsd": 0.05856235454545452, "meanCachedTurnUsd": 0.05856235454545452, "coldUsdPerCorrect": 0.06528911148648646, "cachedUsdPerCorrect": 0.06528911148648646 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-sol@low", "runs": 171, "passRate": 0.8421052631578947, "meanColdCallUsd": 0.05400191228070176, "meanCachedTurnUsd": 0.05400191228070176, "coldUsdPerCorrect": 0.06412727083333335, "cachedUsdPerCorrect": 0.06412727083333335 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-terra@low", "runs": 168, "passRate": 0.7261904761904762, "meanColdCallUsd": 0.03187844196428572, "meanCachedTurnUsd": 0.03187844196428572, "coldUsdPerCorrect": 0.04389818237704919, "cachedUsdPerCorrect": 0.04389818237704919 }, { "condition": "Feliz (bare)", "conditionId": "feliz_bare", "provider": "xAI Grok", "providerModelId": "grok-4.5", "runs": 183, "passRate": 0.6666666666666666, "meanColdCallUsd": 0.01593944043715847, "meanCachedTurnUsd": 0.01593944043715847, "coldUsdPerCorrect": 0.023909160655737702, "cachedUsdPerCorrect": 0.023909160655737702 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "runs": 423, "passRate": 0.9905437352245863, "meanColdCallUsd": 0.06475062529550826, "meanCachedTurnUsd": 0.0645662990543735, "coldUsdPerCorrect": 0.06536876968973745, "cachedUsdPerCorrect": 0.06518268377088303 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "Claude (Opus)", "providerModelId": "claude-fable-5@low", "runs": 60, "passRate": 1, "meanColdCallUsd": 0.09870316666666666, "meanCachedTurnUsd": 0.09783683333333333, "coldUsdPerCorrect": 0.09870316666666666, "cachedUsdPerCorrect": 0.09783683333333333 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "Claude (Opus)", "providerModelId": "claude-opus-4-8@low", "runs": 60, "passRate": 0.95, "meanColdCallUsd": 0.05048033333333333, "meanCachedTurnUsd": 0.05004716666666666, "coldUsdPerCorrect": 0.05313719298245614, "cachedUsdPerCorrect": 0.05268122807017544 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "Google Gemini", "providerModelId": "gemini-3.1-pro-preview", "runs": 60, "passRate": 0.9833333333333333, "meanColdCallUsd": 0.07782620000000003, "meanCachedTurnUsd": 0.07782620000000003, "coldUsdPerCorrect": 0.07914528813559325, "cachedUsdPerCorrect": 0.07914528813559325 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "Google Gemini", "providerModelId": "gemini-3.6-flash", "runs": 60, "passRate": 1, "meanColdCallUsd": 0.09138502499999998, "meanCachedTurnUsd": 0.09138502499999998, "coldUsdPerCorrect": 0.09138502499999998, "cachedUsdPerCorrect": 0.09138502499999998 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-sol@low", "runs": 60, "passRate": 1, "meanColdCallUsd": 0.07708537499999998, "meanCachedTurnUsd": 0.07708537499999998, "coldUsdPerCorrect": 0.07708537499999998, "cachedUsdPerCorrect": 0.07708537499999998 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "OpenAI GPT", "providerModelId": "gpt-5.6-terra@low", "runs": 60, "passRate": 1, "meanColdCallUsd": 0.03780327499999999, "meanCachedTurnUsd": 0.03780327499999999, "coldUsdPerCorrect": 0.03780327499999999, "cachedUsdPerCorrect": 0.03780327499999999 }, { "condition": "Dash (Python)", "conditionId": "dash_py", "provider": "xAI Grok", "providerModelId": "grok-4.5", "runs": 63, "passRate": 1, "meanColdCallUsd": 0.02210336507936508, "meanCachedTurnUsd": 0.02210336507936508, "coldUsdPerCorrect": 0.02210336507936508, "cachedUsdPerCorrect": 0.02210336507936508 } ], "costPerCorrectNote": "Expected spend per correct artifact: mean per-call USD over ALL primary cells (failures included; repair cells excluded) / pass rate. Each condition passes its OWN gate - fuaran: strict wire parse AND judge; code baselines: a judged render gate over code never compiled or executed in-harness - so cross-condition comparison is conservative toward the baselines.", "costPerCorrectCaveat": "spend totals undercount whenever attempts were abandoned client-side (timeouts bill the provider but return no usage payload) \u2014 see the harness cost-undercount caveat", "sessionEconomics": { "cohort": "618-stage2-20260807", "excludedTasks": [ { "id": "tier-b-001", "reason": "pre-v3 probe scaffold: identity figures structurally unfair (authoring protocol v3 note)" } ], "arms": [ { "condition": "feliz_bare", "modelPin": "claude-opus-4-8", "cells": 24, "usdAtTurn3": 0.16512339583333333, "cellsAtTurn3": 24, "usdAtTurn5": 0.5593513333333334, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "feliz_bare", "modelPin": "claude-opus-4-8@low", "cells": 24, "usdAtTurn3": 0.13218589583333334, "cellsAtTurn3": 24, "usdAtTurn5": 0.42787908333333335, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "feliz_bare", "modelPin": "gemini-3.1-pro-preview", "cells": 24, "usdAtTurn3": 0.14046783333333335, "cellsAtTurn3": 24, "usdAtTurn5": 0.3589599666666667, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "feliz_bare", "modelPin": "gpt-4o", "cells": 24, "usdAtTurn3": 0.02625416666666667, "cellsAtTurn3": 24, "usdAtTurn5": 0.07226791666666667, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "fuaran", "modelPin": "claude-opus-4-8", "cells": 24, "usdAtTurn3": 0.19215895833333335, "cellsAtTurn3": 24, "usdAtTurn5": 0.34094083333333336, "cellsAtTurn5": 6, "identityMean": 0.9861111111111112, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "fuaran", "modelPin": "claude-opus-4-8@low", "cells": 24, "usdAtTurn3": 0.15023958333333334, "cellsAtTurn3": 24, "usdAtTurn5": 0.3338925, "cellsAtTurn5": 6, "identityMean": 0.986111111111111, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "fuaran", "modelPin": "gemini-3.1-pro-preview", "cells": 24, "usdAtTurn3": 0.19021734166666668, "cellsAtTurn3": 24, "usdAtTurn5": 0.3297098666666667, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "fuaran", "modelPin": "gpt-4o", "cells": 24, "usdAtTurn3": 0.0935509375, "cellsAtTurn3": 24, "usdAtTurn5": 0.17769875, "cellsAtTurn5": 6, "identityMean": 0.9583333333333335, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "jsx_shadcn", "modelPin": "claude-opus-4-8", "cells": 24, "usdAtTurn3": 0.17562533333333336, "cellsAtTurn3": 24, "usdAtTurn5": 0.5451363333333333, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "jsx_shadcn", "modelPin": "claude-opus-4-8@low", "cells": 24, "usdAtTurn3": 0.14159591666666668, "cellsAtTurn3": 24, "usdAtTurn5": 0.443808, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" }, { "condition": "jsx_shadcn", "modelPin": "gemini-3.1-pro-preview", "cells": 25, "usdAtTurn3": 0.12795984, "cellsAtTurn3": 25, "usdAtTurn5": 0.2658003333333333, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 25, "pricingRetrieved": "2026-07-30" }, { "condition": "jsx_shadcn", "modelPin": "gpt-4o", "cells": 24, "usdAtTurn3": 0.024686770833333333, "cellsAtTurn3": 24, "usdAtTurn5": 0.0650575, "cellsAtTurn5": 6, "identityMean": 1, "identityCells": 24, "pricingRetrieved": "2026-07-30" } ], "note": "Tier-B multi-turn session economics: cumulative billed USD through turn k, mean over sessions that reached turn k (denominator beside each figure); identity = fraction of prompt-named element ids present in both the first and final emission, mean over measured cells. Multi-turn cells publish here only \u2014 parsePassRate remains a single-shot claim." } }