{ "protocol": "analytics-v2", "publicationDate": "2026-11-14", "timeZone": "America/Los_Angeles", "dateWindows": { "current": [ "2026-11-07", "2026-11-13" ], "comparator": [ "2026-10-31", "2026-11-06" ] }, "referencePreparedAt": "2026-11-14T18:04:02.721Z", "referenceDrift": { "checkedAt": "2026-11-14T20:52:40.022Z", "changedReports": [ "current.totals", "current.daily", "current.acquisition", "current.devices" ], "currentWeekTotalsBefore": { "sessions": 7800, "engagedSessions": 4776, "totalUsers": 4159 }, "currentWeekTotalsAfter": { "sessions": 7780, "engagedSessions": 4794, "totalUsers": 4159 }, "currentLastDayBefore": { "sessions": 1151, "engagedSessions": 676 }, "currentLastDayAfter": { "sessions": 1152, "engagedSessions": 674 }, "affectedFinalAnswer": { "model": "openai/gpt-6-astra", "attempt": 3 }, "review": "Astra attempt3 used the revised weekly and daily totals, and correctly cited its own returned device rows. The later reference device report also changed. Its claims were independently checked against the rows it received; no awarded points changed. Comparator reports and current named-event/commerce references matched at the checked precision. The revisions did not reverse the observed engagement decline or reported lifecycle/attribution gaps. This is a date-pinned live-report comparison, not an identical immutable snapshot. We cannot measure what every model would have answered under a different snapshot." }, "configuration": { "repetitions": 3, "investigationResponses": 4, "reportsPerInvestigationResponse": 3, "optionalFinalResponse": 1, "maxCompletionTokensPerResponse": 8192, "temperature": 0.3, "reasoningRoutingAndCaching": "OpenRouter/provider defaults", "promptSha256": "2d106697a20867ce78a573a318021d0ebb1f6a4c59ebc741731e3f985775df6d", "rubricSha256": "5e44c6f8d575757a00fb4f442de5fe25674d23a84fd4b7ca8b3be0cc667c07c5", "reportViewSha256": "7fa96b17faed4d6ac43ea7900d37f55910c0e4a2cef5db6508bbda20a4b152ed" }, "units": { "evidenceScore": "Points out of 100; null means unscored infrastructure failure", "providerSeconds": "Sum of provider-response durations within this attempt; excludes GA4 requests", "wallSeconds": "Elapsed end-to-end attempt time", "reportedCostUsd": "USD from provider usage when every saved response has cost metadata; otherwise null", "modelMeans": "Mean across scored attempts, including genuine no-final zeros; infrastructure failures excluded" }, "plannedAttempts": 27, "finalAnswers": 15, "scoredAttempts": 21, "knownScoredCostUsd": 4.4819892, "knownAdditionalCostUsd": 0.081763875, "unknownInterruptedCharge": true, "criteria": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 6, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 8, "Evidence limitations": 6 }, "limitations": [ "One synthetic demo property; this does not measure production false-positive rates.", "Three planned attempts per model; provider errors are unscored and genuine no-final attempts score zero.", "A fixed JSON report interface and response budget form part of the task; reasoning and routing use provider defaults.", "The acquisition report only accepts sessions/engagedSessions and no second dimension; this restriction was not fully described in the model-facing guide.", "Evidence scores are separate from historical GA4 field scores; compare within one batch first.", "Dates are pinned, but live GA4 reports may change while processing. Claims were graded against each attempt's returned rows.", "AI-assisted rubric judgments include subjective interpretation; numerical claims were separately checked against report rows." ], "models": [ { "id": "openai/gpt-6-astra", "name": "GPT-6 Astra", "provider": "OpenAI", "scores": [ 100, 100, 100 ], "score": 100, "finalCount": 3, "time": 81.80066666666667, "wallTime": 84.81233333333334, "cost": 0.27302883333333333 }, { "id": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "provider": "OpenAI", "scores": [ 90, 97, 99 ], "score": 95.33333333333333, "finalCount": 3, "time": 67.04366666666665, "wallTime": 69.24466666666667, "cost": 0.04872373333333333 }, { "id": "openai/gpt-6.1-terra", "name": "GPT-6.1 Terra", "provider": "OpenAI", "scores": [ 85, 89, 85 ], "score": 86.33333333333333, "finalCount": 3, "time": 41.842666666666666, "wallTime": 43.68833333333333, "cost": 0.020669166666666666 }, { "id": "x-ai/grok-4.7", "name": "Grok 4.7", "provider": "xAI", "scores": [ 82, 82, 79 ], "score": 81, "finalCount": 3, "time": 194.8983333333333, "wallTime": 197.06033333333335, "cost": 0.11136733333333333 }, { "id": "anthropic/claude-opus-5.5", "name": "Claude Opus 5.5", "provider": "Anthropic", "scores": [ 85, 74, 74 ], "score": 77.66666666666667, "finalCount": 3, "time": 79.46366666666667, "wallTime": 80.96666666666667, "cost": 0.22044666666666668 }, { "id": "anthropic/claude-fable-5.2", "name": "Claude Fable 5.2", "provider": "Anthropic", "scores": [ 0, 0, 0 ], "score": 0, "finalCount": 0, "time": 132.13733333333334, "wallTime": 134.78300000000002, "cost": 0.702048 }, { "id": "qwen/qwen3.8-max-0902", "name": "Qwen3.8 Max (0902)", "provider": "Qwen", "scores": [ 0, 0, 0 ], "score": 0, "finalCount": 0, "time": 524.2143333333333, "wallTime": 527.2636666666667, "cost": 0.11771266666666667 }, { "id": "google/gemini-4-carbon", "name": "Gemini 4 Carbon", "provider": "Google", "scores": [ null, null, null ], "score": null, "finalCount": 0, "time": null, "wallTime": null, "cost": null }, { "id": "google/gemini-4-hydrogen", "name": "Gemini 4 Hydrogen", "provider": "Google", "scores": [ null, null, null ], "score": null, "finalCount": 0, "time": null, "wallTime": null, "cost": null } ], "attempts": [ { "model": "google/gemini-4-hydrogen", "attempt": 1, "status": "provider_error", "evidenceScore": null, "criterionScores": null, "conclusion": null, "confidence": null, "providerSeconds": null, "wallSeconds": null, "reportedCostUsd": null, "successfulReports": 0, "totalReports": 0 }, { "model": "google/gemini-4-hydrogen", "attempt": 2, "status": "provider_error", "evidenceScore": null, "criterionScores": null, "conclusion": null, "confidence": null, "providerSeconds": null, "wallSeconds": null, "reportedCostUsd": null, "successfulReports": 0, "totalReports": 0 }, { "model": "google/gemini-4-hydrogen", "attempt": 3, "status": "provider_error", "evidenceScore": null, "criterionScores": null, "conclusion": null, "confidence": null, "providerSeconds": null, "wallSeconds": null, "reportedCostUsd": null, "successfulReports": 0, "totalReports": 0 }, { "model": "google/gemini-4-carbon", "attempt": 1, "status": "provider_error", "evidenceScore": null, "criterionScores": null, "conclusion": null, "confidence": null, "providerSeconds": null, "wallSeconds": null, "reportedCostUsd": null, "successfulReports": 0, "totalReports": 0 }, { "model": "google/gemini-4-carbon", "attempt": 2, "status": "provider_error", "evidenceScore": null, "criterionScores": null, "conclusion": null, "confidence": null, "providerSeconds": null, "wallSeconds": null, "reportedCostUsd": null, "successfulReports": 0, "totalReports": 0 }, { "model": "google/gemini-4-carbon", "attempt": 3, "status": "provider_error", "evidenceScore": null, "criterionScores": null, "conclusion": null, "confidence": null, "providerSeconds": null, "wallSeconds": null, "reportedCostUsd": null, "successfulReports": 0, "totalReports": 0 }, { "model": "anthropic/claude-fable-5.2", "attempt": 1, "status": "no_final", "evidenceScore": 0, "criterionScores": { "Weekly session totals": 0, "Weekly distinct users": 0, "Sessions per user": 0, "Engagement": 0, "Temporal localization": 0, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 0, "Commerce reconciliation": 0, "Signup identification": 0, "Competing explanation": 0, "Discriminating follow-up": 0, "Conclusion and confidence": 0, "Evidence limitations": 0 }, "conclusion": null, "confidence": null, "providerSeconds": 121.597, "wallSeconds": 124.128, "reportedCostUsd": 0.692608, "successfulReports": 9, "totalReports": 12 }, { "model": "anthropic/claude-fable-5.2", "attempt": 2, "status": "no_final", "evidenceScore": 0, "criterionScores": { "Weekly session totals": 0, "Weekly distinct users": 0, "Sessions per user": 0, "Engagement": 0, "Temporal localization": 0, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 0, "Commerce reconciliation": 0, "Signup identification": 0, "Competing explanation": 0, "Discriminating follow-up": 0, "Conclusion and confidence": 0, "Evidence limitations": 0 }, "conclusion": null, "confidence": null, "providerSeconds": 134.333, "wallSeconds": 137.189, "reportedCostUsd": 0.6970000000000001, "successfulReports": 10, "totalReports": 12 }, { "model": "anthropic/claude-fable-5.2", "attempt": 3, "status": "no_final", "evidenceScore": 0, "criterionScores": { "Weekly session totals": 0, "Weekly distinct users": 0, "Sessions per user": 0, "Engagement": 0, "Temporal localization": 0, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 0, "Commerce reconciliation": 0, "Signup identification": 0, "Competing explanation": 0, "Discriminating follow-up": 0, "Conclusion and confidence": 0, "Evidence limitations": 0 }, "conclusion": null, "confidence": null, "providerSeconds": 140.482, "wallSeconds": 143.032, "reportedCostUsd": 0.7165360000000001, "successfulReports": 11, "totalReports": 12 }, { "model": "openai/gpt-6.1-terra", "attempt": 1, "status": "complete", "evidenceScore": 85, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 0, "Session and event reconciliation": 6, "Commerce reconciliation": 2, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 7, "Evidence limitations": 6 }, "conclusion": "inconclusive", "confidence": 85, "providerSeconds": 41.902, "wallSeconds": 44.265, "reportedCostUsd": 0.02045425, "successfulReports": 7, "totalReports": 9 }, { "model": "openai/gpt-6.1-terra", "attempt": 2, "status": "complete", "evidenceScore": 89, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 4, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 7, "Evidence limitations": 6 }, "conclusion": "inconclusive", "confidence": 70, "providerSeconds": 44.131, "wallSeconds": 45.89, "reportedCostUsd": 0.02169345, "successfulReports": 7, "totalReports": 9 }, { "model": "openai/gpt-6.1-terra", "attempt": 3, "status": "complete", "evidenceScore": 85, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 0, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 7, "Evidence limitations": 6 }, "conclusion": "inconclusive", "confidence": 55, "providerSeconds": 39.495, "wallSeconds": 40.91, "reportedCostUsd": 0.0198598, "successfulReports": 6, "totalReports": 6 }, { "model": "openai/gpt-6.1-sol", "attempt": 1, "status": "complete", "evidenceScore": 90, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 4, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 8, "Evidence limitations": 6 }, "conclusion": "synthetic", "confidence": 65, "providerSeconds": 71.841, "wallSeconds": 73.79, "reportedCostUsd": 0.0494068, "successfulReports": 7, "totalReports": 9 }, { "model": "openai/gpt-6.1-sol", "attempt": 2, "status": "complete", "evidenceScore": 97, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 3, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 8, "Evidence limitations": 6 }, "conclusion": "synthetic", "confidence": 72, "providerSeconds": 63.857, "wallSeconds": 66.188, "reportedCostUsd": 0.0454604, "successfulReports": 10, "totalReports": 12 }, { "model": "openai/gpt-6.1-sol", "attempt": 3, "status": "complete", "evidenceScore": 99, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 6, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 7, "Evidence limitations": 6 }, "conclusion": "inconclusive", "confidence": 80, "providerSeconds": 65.433, "wallSeconds": 67.756, "reportedCostUsd": 0.051303999999999995, "successfulReports": 11, "totalReports": 12 }, { "model": "anthropic/claude-opus-5.5", "attempt": 1, "status": "complete", "evidenceScore": 85, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 0, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 7, "Evidence limitations": 6 }, "conclusion": "inconclusive", "confidence": 55, "providerSeconds": 72.552, "wallSeconds": 73.879, "reportedCostUsd": 0.200168, "successfulReports": 6, "totalReports": 6 }, { "model": "anthropic/claude-opus-5.5", "attempt": 2, "status": "complete", "evidenceScore": 74, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 0, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 7, "Conclusion and confidence": 6, "Evidence limitations": 3 }, "conclusion": "inconclusive", "confidence": 55, "providerSeconds": 70.443, "wallSeconds": 71.825, "reportedCostUsd": 0.197372, "successfulReports": 6, "totalReports": 6 }, { "model": "anthropic/claude-opus-5.5", "attempt": 3, "status": "complete", "evidenceScore": 74, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 0, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 7, "Conclusion and confidence": 6, "Evidence limitations": 3 }, "conclusion": "inconclusive", "confidence": 60, "providerSeconds": 95.396, "wallSeconds": 97.196, "reportedCostUsd": 0.26380000000000003, "successfulReports": 7, "totalReports": 9 }, { "model": "x-ai/grok-4.7", "attempt": 1, "status": "complete", "evidenceScore": 82, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 7, "Conclusion and confidence": 6, "Evidence limitations": 3 }, "conclusion": "synthetic", "confidence": 70, "providerSeconds": 190.089, "wallSeconds": 192.087, "reportedCostUsd": 0.1067, "successfulReports": 10, "totalReports": 12 }, { "model": "x-ai/grok-4.7", "attempt": 2, "status": "complete", "evidenceScore": 82, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 7, "Conclusion and confidence": 6, "Evidence limitations": 3 }, "conclusion": "synthetic", "confidence": 74, "providerSeconds": 211.566, "wallSeconds": 213.988, "reportedCostUsd": 0.12151, "successfulReports": 8, "totalReports": 12 }, { "model": "x-ai/grok-4.7", "attempt": 3, "status": "complete", "evidenceScore": 79, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 4, "Conclusion and confidence": 6, "Evidence limitations": 3 }, "conclusion": "synthetic", "confidence": 71, "providerSeconds": 183.04, "wallSeconds": 185.106, "reportedCostUsd": 0.10589199999999999, "successfulReports": 8, "totalReports": 12 }, { "model": "qwen/qwen3.8-max-0902", "attempt": 1, "status": "no_final", "evidenceScore": 0, "criterionScores": { "Weekly session totals": 0, "Weekly distinct users": 0, "Sessions per user": 0, "Engagement": 0, "Temporal localization": 0, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 0, "Commerce reconciliation": 0, "Signup identification": 0, "Competing explanation": 0, "Discriminating follow-up": 0, "Conclusion and confidence": 0, "Evidence limitations": 0 }, "conclusion": null, "confidence": null, "providerSeconds": 546.501, "wallSeconds": 549.697, "reportedCostUsd": 0.12305, "successfulReports": 7, "totalReports": 12 }, { "model": "qwen/qwen3.8-max-0902", "attempt": 2, "status": "no_final", "evidenceScore": 0, "criterionScores": { "Weekly session totals": 0, "Weekly distinct users": 0, "Sessions per user": 0, "Engagement": 0, "Temporal localization": 0, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 0, "Commerce reconciliation": 0, "Signup identification": 0, "Competing explanation": 0, "Discriminating follow-up": 0, "Conclusion and confidence": 0, "Evidence limitations": 0 }, "conclusion": null, "confidence": null, "providerSeconds": 616.468, "wallSeconds": 619.424, "reportedCostUsd": 0.1209, "successfulReports": 7, "totalReports": 12 }, { "model": "qwen/qwen3.8-max-0902", "attempt": 3, "status": "no_final", "evidenceScore": 0, "criterionScores": { "Weekly session totals": 0, "Weekly distinct users": 0, "Sessions per user": 0, "Engagement": 0, "Temporal localization": 0, "Device check": 0, "Acquisition check": 0, "Session and event reconciliation": 0, "Commerce reconciliation": 0, "Signup identification": 0, "Competing explanation": 0, "Discriminating follow-up": 0, "Conclusion and confidence": 0, "Evidence limitations": 0 }, "conclusion": null, "confidence": null, "providerSeconds": 409.674, "wallSeconds": 412.67, "reportedCostUsd": 0.10918800000000001, "successfulReports": 8, "totalReports": 12 }, { "model": "openai/gpt-6-astra", "attempt": 1, "status": "complete", "evidenceScore": 100, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 6, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 8, "Evidence limitations": 6 }, "conclusion": "synthetic", "confidence": 78, "providerSeconds": 67.81, "wallSeconds": 71.093, "reportedCostUsd": 0.26642750000000004, "successfulReports": 11, "totalReports": 12 }, { "model": "openai/gpt-6-astra", "attempt": 2, "status": "complete", "evidenceScore": 100, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 6, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 8, "Evidence limitations": 6 }, "conclusion": "synthetic", "confidence": 70, "providerSeconds": 95.764, "wallSeconds": 98.562, "reportedCostUsd": 0.29230500000000004, "successfulReports": 11, "totalReports": 12 }, { "model": "openai/gpt-6-astra", "attempt": 3, "status": "complete", "evidenceScore": 100, "criterionScores": { "Weekly session totals": 6, "Weekly distinct users": 6, "Sessions per user": 6, "Engagement": 6, "Temporal localization": 8, "Device check": 6, "Acquisition check": 6, "Session and event reconciliation": 8, "Commerce reconciliation": 8, "Signup identification": 10, "Competing explanation": 8, "Discriminating follow-up": 8, "Conclusion and confidence": 8, "Evidence limitations": 6 }, "conclusion": "synthetic", "confidence": 70, "providerSeconds": 81.828, "wallSeconds": 84.782, "reportedCostUsd": 0.260354, "successfulReports": 10, "totalReports": 12 } ] }