{
  "schemaVersion": 2,
  "snapshotDate": "2026-11-10",
  "humanReviewed": false,
  "omittedModels": 12,
  "models": [
    {
      "modelId": "audio8-asr-infinite",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 36,
          "attempted": 20,
          "valid": 20,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "language-support-unverified",
            "worker-stopped-abnormally",
            "retry-included",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 14,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.02857142857142857,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "en"
              ],
              "limitations": []
            }
          ]
        },
        {
          "id": "stt-2",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 4,
          "attempted": 4,
          "valid": 4,
          "languages": [
            "de"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "language-support-unverified",
            "english-token-on-german-audio",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 76,
              "denominator": 109,
              "denominatorBasis": "reference-words",
              "value": 0.6972477064220184,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 196,
              "denominator": 657,
              "denominatorBasis": "reference-characters",
              "value": 0.2983257229832572,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "autotrustjev-27b",
      "category": "decision",
      "status": "smoke",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "cpu",
            "precision": "bf16"
          },
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "smoke-only",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "autotrustjev-27b-vl",
      "category": "decision",
      "status": "smoke",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "cpu",
            "precision": "bf16"
          },
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "smoke-only",
            "no-quality-assessment",
            "text-only-input"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "cartesia-ink-2",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "public-demo",
          "variant": "default",
          "expected": 20,
          "attempted": 20,
          "valid": 15,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "model-identity-limited",
            "no-silence-assessment",
            "provider-request-count-unverified",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 12,
              "denominator": 447,
              "denominatorBasis": "reference-words",
              "value": 0.026845637583892617,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 24,
              "denominator": 2350,
              "denominatorBasis": "reference-characters",
              "value": 0.010212765957446808,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "numerator": 0,
              "denominator": 0,
              "denominatorBasis": "assessable-silence",
              "value": null,
              "languages": [
                "en"
              ],
              "limitations": [
                "no-silence-assessment"
              ]
            }
          ]
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-silence-assessment",
            "provider-request-count-unverified",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "cartesia-sonic-3-6",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "public-demo",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 8,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.049689440993788817,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 34,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.038159371492704826,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 7,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.04375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 24,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.030226700251889168,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 15,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.04672897196261682,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 58,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.034421364985163204,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 2,
          "attempted": 2,
          "valid": 2,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "clef",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "cloudflare",
          "variant": "default",
          "execution": {
            "runtime": "hosted",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 345,
            "p95Ms": 700,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 55,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 29,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.9666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 38,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.95,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 2,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.038461538461538464,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 1,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "clef-flash",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "cloudflare",
          "variant": "default",
          "execution": {
            "runtime": "hosted",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 599,
            "p95Ms": 1315,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "nearest-rank"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 51,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.85,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 26,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.8666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 36,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.9,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 1,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.019230769230769232,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 1,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "clm-v0.1-8b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 661.5,
            "p95Ms": 1363,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 21,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.35,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 5,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.4166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 5,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 15,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.375,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 30,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.5769230769230769,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 2,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "confucius4-r2t2",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "public-demo",
          "variant": "default",
          "expected": 36,
          "attempted": 35,
          "valid": 30,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "technical-only",
            "limited-inputs",
            "unreviewed-reference",
            "model-identity-limited",
            "no-silence-assessment",
            "ui-final-unverified"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 58,
              "denominator": 518,
              "denominatorBasis": "reference-words",
              "value": 0.11196911196911197,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "ui-final-unverified"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 116,
              "denominator": 3074,
              "denominatorBasis": "reference-characters",
              "value": 0.03773584905660377,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "ui-final-unverified"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 10,
              "denominator": 471,
              "denominatorBasis": "reference-words",
              "value": 0.021231422505307854,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "ui-final-unverified"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 29,
              "denominator": 2481,
              "denominatorBasis": "reference-characters",
              "value": 0.011688835147118097,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "ui-final-unverified"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 68,
              "denominator": 989,
              "denominatorBasis": "reference-words",
              "value": 0.06875631951466127,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "ui-final-unverified"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 145,
              "denominator": 5555,
              "denominatorBasis": "reference-characters",
              "value": 0.026102610261026102,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "ui-final-unverified"
              ]
            },
            {
              "kind": "stt-silence-text",
              "numerator": 0,
              "denominator": 0,
              "denominatorBasis": "assessable-silence",
              "value": null,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "no-silence-assessment",
                "ui-final-unverified"
              ]
            }
          ]
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-silence-assessment",
            "ui-final-unverified",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "decider-0.8b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 3047.5,
            "p95Ms": 3945,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 27,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.45,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 1,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.08333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 18,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.6,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 26,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.65,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 2,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.038461538461538464,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 4,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "decider-2b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 129,
            "p95Ms": 174,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 40,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 26,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.8666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 29,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.725,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 4,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "decider-2b-vision",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 156.5,
            "p95Ms": 301,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "text-only-input"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 34,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.5666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 20,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 30,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 2,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.038461538461538464,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 4,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "decider-35b-a3b",
      "category": "decision",
      "status": "partial",
      "cohorts": [
        {
          "id": "decision-1",
          "protocol": "decision",
          "route": "public-demo",
          "variant": "default",
          "execution": {
            "runtime": "hosted",
            "precision": "unknown"
          },
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "unreviewed-reference",
            "partial-coverage",
            "no-quality-assessment",
            "model-identity-limited"
          ],
          "metrics": []
        },
        {
          "id": "decision-2",
          "protocol": "decision",
          "route": "public-demo",
          "variant": "default",
          "execution": {
            "runtime": "hosted",
            "precision": "unknown"
          },
          "expected": 130,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "partial-coverage",
            "no-quality-assessment",
            "model-identity-limited"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "decider-35b-a3b-nvfp4",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "nvfp4"
          },
          "timing": {
            "p50Ms": 245,
            "p95Ms": 368,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu-forward-only",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "hardware-quantization-specific"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 56,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 30,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 38,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.95,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 2,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "decider-4b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 231.5,
            "p95Ms": 313,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 51,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.85,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 25,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 35,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.875,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 1,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.019230769230769232,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 2,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "deepgram-aura-2",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "cloudflare",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 13,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.040625,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 49,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.030856423173803528,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "deepgram-flux-general-en",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "cloudflare",
          "variant": "default",
          "expected": 18,
          "attempted": 18,
          "valid": 18,
          "languages": [
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 24,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.04897959183673469,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "stream-any-text",
              "numerator": 0,
              "denominator": 2,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "deepgram-nova-3",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "cloudflare",
          "variant": "default",
          "timing": {
            "p50Ms": 1180.5,
            "p95Ms": 2849,
            "sampleCount": 36,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "median-and-nearest-rank"
          },
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 47,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.08545454545454545,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 85,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.026057633353770693,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 10,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.02040816326530612,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 17,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.006589147286821705,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 57,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.05480769230769231,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 102,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.017459774049982883,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "eleven-flash-v2-5",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 8,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.024844720496894408,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 24,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.013468013468013467,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 19,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.059375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 57,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.03589420654911839,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 27,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.04205607476635514,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 81,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.024035608308605343,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "eleven-realtime-v1-max",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "gateway",
          "variant": "default",
          "expected": 6,
          "attempted": 6,
          "valid": 5,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "technical-only",
            "limited-inputs",
            "model-identity-limited"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "gateway",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "eleven-realtime-v1-mini",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "gateway",
          "variant": "default",
          "expected": 6,
          "attempted": 6,
          "valid": 6,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "gateway",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "eleven-text-to-sound-v2",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "fal",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "eleven-text-to-sound-v3",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "fal",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "eleven-v3",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.015527950310559006,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 18,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.010101010101010102,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 12,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.0375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 44,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.027707808564231738,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 17,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.0264797507788162,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 62,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.018397626112759646,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "eleven-v3-conversational",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.015527950310559006,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 21,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.011784511784511785,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 10,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 40,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 15,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.02336448598130841,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 61,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.018100890207715135,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "eleven-v4",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "fal",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 6,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.018633540372670808,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 32,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.017957351290684626,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 20,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.0625,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 50,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.031486146095717885,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 26,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.040498442367601244,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 82,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.02433234421364985,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "eleven-v4-turbo",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.015527950310559006,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 21,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.011784511784511785,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 18,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.05625,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 46,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.028967254408060455,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 23,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.03582554517133956,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 67,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.019881305637982197,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "falcon-asr-1.6b",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 6,
          "attempted": 6,
          "valid": 6,
          "languages": [
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment",
            "limited-language"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 2,
          "attempted": 2,
          "valid": 2,
          "languages": [
            "ar",
            "en"
          ],
          "limitations": [
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "fish-audio-s2-1-pro",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 3,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.018633540372670808,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 7,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.007856341189674524,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 6,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.0375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 23,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.028967254408060455,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 9,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.028037383177570093,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 30,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.017804154302670624,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "fish-audio-s2-pro",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 3,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.018633540372670808,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 7,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.007856341189674524,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 20,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 8,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.024922118380062305,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 27,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.016023738872403562,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gemini-3-1-flash-tts-preview",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 7,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.021739130434782608,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 36,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.020202020202020204,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 14,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.04375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 43,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.02707808564231738,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 21,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.03271028037383177,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 79,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.02344213649851632,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gemini-3-5-live-translate-preview",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "vertex",
          "variant": "default",
          "expected": 2,
          "attempted": 2,
          "valid": 2,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "gemini-3-5-transcribe",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 38,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.03653846153846154,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 91,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.015576857240671003,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "gemini-3-5-transcribe-live",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "gateway",
          "variant": "default",
          "expected": 36,
          "attempted": 35,
          "valid": 35,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "model-identity-limited",
            "limited-streaming"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 43,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.07818181818181819,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 10,
              "denominator": 426,
              "denominatorBasis": "reference-words",
              "value": 0.023474178403755867,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 53,
              "denominator": 976,
              "denominatorBasis": "reference-words",
              "value": 0.05430327868852459,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "stream-any-text",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "gemini-3-8-live",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "gateway",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "peer-close-unverified",
            "text-only-input",
            "limited-language"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "vertex",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "peer-close-unverified",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "gemini-3-8-live-extended-thinking",
      "category": "audio",
      "status": "smoke",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "gateway",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "de"
          ],
          "limitations": [
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "text-only-input",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "gemini-3.8-flash-lite-tts",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 9,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.027950310559006212,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 28,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.015712682379349047,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 14,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.04375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 43,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.02707808564231738,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 23,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.03582554517133956,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 71,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.021068249258160237,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gemini-3.8-flash-tts",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.015527950310559006,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 12,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.006734006734006734,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 18,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.05625,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 51,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.03211586901763224,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 23,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.03582554517133956,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 63,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.018694362017804153,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "glide",
      "category": "decision",
      "status": "partial",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "cpu",
            "precision": "bf16"
          },
          "expected": 130,
          "attempted": 32,
          "valid": 29,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "partial-coverage",
            "no-quality-assessment",
            "text-only-input"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "gliner2-5-decide",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 208.5,
            "p95Ms": 286,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 26,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.43333333333333335,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 5,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.4166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 15,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 10,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 10,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.19230769230769232,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 3,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.375,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gliner2-5-decide-1b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "cpu",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 885,
            "p95Ms": 2148,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 15,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 1,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.08333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 1,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.08333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 7,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.23333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 16,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.4,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 9,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.17307692307692307,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 6,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gliner2-5-multi-decide",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 75,
            "p95Ms": 111,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 15,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 1,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.08333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 17,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.5666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 10,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 10,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.19230769230769232,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 5,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.625,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-4o-mini-transcribe",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 43,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.041346153846153845,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 86,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.014720985963711058,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 1,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-4o-mini-tts",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "direct",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.031055900621118012,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 21,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.02356902356902357,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 11,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.06875,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 46,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.05793450881612091,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 16,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.04984423676012461,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 67,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.039762611275964393,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-live-1",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "direct",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "synthetic-speech",
            "no-naturalness-evaluation",
            "post-hoc-exclusions",
            "literal-translation-reference",
            "provider-transcript-proxy"
          ],
          "metrics": [
            {
              "kind": "technical-dialogue-correct",
              "numerator": 3,
              "denominator": 6,
              "denominatorBasis": "assessable-dialogue",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "post-hoc-exclusions",
                "literal-translation-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-live-transcribe",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "direct",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 47,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.08545454545454545,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 108,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.03310852237890864,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 10,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.02040816326530612,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 17,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.006589147286821705,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 57,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.05480769230769231,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 125,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.02139678192399863,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-realtime-2-1",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "direct",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "synthetic-speech",
            "no-naturalness-evaluation",
            "post-hoc-exclusions",
            "literal-translation-reference",
            "provider-transcript-proxy"
          ],
          "metrics": [
            {
              "kind": "technical-dialogue-correct",
              "numerator": 5,
              "denominator": 6,
              "denominatorBasis": "assessable-dialogue",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "post-hoc-exclusions",
                "literal-translation-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-realtime-2-1-mini",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "direct",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 7,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "peer-close-unverified",
            "synthetic-speech",
            "no-naturalness-evaluation",
            "post-hoc-exclusions",
            "literal-translation-reference",
            "provider-transcript-proxy"
          ],
          "metrics": [
            {
              "kind": "technical-dialogue-correct",
              "numerator": 5,
              "denominator": 5,
              "denominatorBasis": "assessable-dialogue",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "post-hoc-exclusions",
                "literal-translation-reference"
              ]
            }
          ]
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "direct",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "peer-close-unverified",
            "synthetic-speech",
            "no-naturalness-evaluation"
          ],
          "metrics": []
        },
        {
          "id": "technical-3",
          "protocol": "technical",
          "route": "direct",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "de"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "peer-close-unverified",
            "repeated-inputs",
            "synthetic-speech",
            "no-naturalness-evaluation",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "gpt-realtime-translate",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "direct",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "synthetic-speech",
            "no-naturalness-evaluation",
            "post-hoc-exclusions",
            "literal-translation-reference",
            "provider-transcript-proxy"
          ],
          "metrics": [
            {
              "kind": "technical-dialogue-correct",
              "numerator": 0,
              "denominator": 0,
              "denominatorBasis": "assessable-dialogue",
              "value": null,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "post-hoc-exclusions",
                "literal-translation-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-realtime-whisper",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "direct",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 42,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.07636363636363637,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 86,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.026364193746167996,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 9,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.018367346938775512,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 17,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.006589147286821705,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 51,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.04903846153846154,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 103,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.01763094830537487,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "gpt-transcribe",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "direct",
          "variant": "default",
          "timing": {
            "p50Ms": 1154,
            "p95Ms": 1974,
            "sampleCount": 36,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "median-and-nearest-rank"
          },
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 37,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.06727272727272728,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 79,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.02421827099938688,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 8,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.0163265306122449,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 21,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.00813953488372093,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 45,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.04326923076923077,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 100,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.017117425539198903,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "granite-speech-5-0-470m-turboctc",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 49,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.0890909090909091,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 139,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.04261189454322502,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 6,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.012244897959183673,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 12,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.004651162790697674,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 55,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.052884615384615384,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 151,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.025847312564190345,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 4,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "index-echo-s2st-2b",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 3,
          "attempted": 3,
          "valid": 2,
          "languages": [
            "zh",
            "en",
            "es"
          ],
          "limitations": [
            "partial-coverage",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "target-language-unverified",
            "requested-audio-languages",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "index-echo-s2st-9b",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 3,
          "attempted": 3,
          "valid": 2,
          "languages": [
            "zh",
            "en",
            "es"
          ],
          "limitations": [
            "partial-coverage",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "target-language-unverified",
            "requested-audio-languages",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "index-echo-s2tt-2b",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "translation-1",
          "protocol": "translation",
          "route": "local",
          "variant": "default",
          "expected": 9,
          "attempted": 9,
          "valid": 9,
          "languages": [
            "zh",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "unreviewed-reference",
            "literal-translation-reference"
          ],
          "metrics": [
            {
              "kind": "translation-literal-wer",
              "direction": {
                "from": "zh",
                "to": "en"
              },
              "numerator": 37,
              "denominator": 51,
              "denominatorBasis": "reference-words",
              "value": 0.7254901960784313,
              "languages": [
                "zh",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "literal-translation-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "index-echo-s2tt-9b",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "target-language-unverified"
          ],
          "metrics": []
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "public-demo",
          "variant": "default",
          "expected": 6,
          "attempted": 6,
          "valid": 6,
          "languages": [
            "zh",
            "en",
            "ja",
            "es"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "target-language-unverified",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "intern-decision-0-8b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 198.5,
            "p95Ms": 642,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 36,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.6,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 20,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 31,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.775,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 2,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.038461538461538464,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 4,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "intern-decision-2b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 290,
            "p95Ms": 575,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 39,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.65,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 22,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.7333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 31,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.775,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 3,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.057692307692307696,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 2,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "intern-decision-4b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 420.5,
            "p95Ms": 821,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 55,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 27,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.9,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 37,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.925,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 1,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "jeff-1",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 274,
            "p95Ms": 750,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "base-revision-unattested"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 42,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.7,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 23,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.7666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 27,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.675,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 3,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.375,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "jev",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "direct",
          "variant": "default",
          "execution": {
            "runtime": "hosted",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 334,
            "p95Ms": 1030,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "nearest-rank"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 56,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 30,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 37,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.925,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 2,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "julia-1",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 41.5,
            "p95Ms": 60,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 17,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.2833333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 13,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.43333333333333335,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 16,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.4,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 7,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.1346153846153846,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 6,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "kev-0.5b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 168,
            "p95Ms": 247,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 23,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.38333333333333336,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 1,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.08333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 5,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.4166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 16,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.5333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 13,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.325,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 10,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.19230769230769232,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 8,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "kev-0.8b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 2882.5,
            "p95Ms": 3439,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 30,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 18,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.6,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 30,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 7,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.1346153846153846,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 0,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "kev-4b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "openrouter",
          "variant": "default",
          "execution": {
            "runtime": "hosted",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 529,
            "p95Ms": 812,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "median-and-nearest-rank"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 47,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.7833333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 8,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 27,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.9,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 35,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.875,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 5,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.09615384615384616,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 0,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "kev-9b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "mixed"
          },
          "timing": {
            "p50Ms": 620,
            "p95Ms": 1233,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 56,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 30,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 34,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.85,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 0,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "kokoro-82m",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "local",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 20,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "laya",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 146.5,
            "p95Ms": 222,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 24,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.4,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 5,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.4166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 21,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.7,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 21,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.525,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 8,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "laya-multilingual",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "cpu",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 53,
            "p95Ms": 78,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "nearest-rank"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 22,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.36666666666666664,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 13,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.43333333333333335,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 16,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.4,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 2,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.038461538461538464,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 7,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.875,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "laya-typed-decisions",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 146.5,
            "p95Ms": 222,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 24,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.4,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 5,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.4166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 15,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 25,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.625,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 3,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.057692307692307696,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 7,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.875,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "lev",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 1255,
            "p95Ms": 1812,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "base-revision-unattested"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 49,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.8166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 25,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 34,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.85,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 4,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.07692307692307693,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 1,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "lfm3-audio-2b",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "mlx",
            "precision": "bf16"
          },
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 57,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.10363636363636364,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 134,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.041079092581238506,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 14,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.02857142857142857,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 25,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.009689922480620155,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 71,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.06826923076923076,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 159,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.02721670660732626,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 4,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "liquid-d1-2b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp16"
          },
          "timing": {
            "p50Ms": 19,
            "p95Ms": 20,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 51,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.85,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 7,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5833333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 29,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.9666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 31,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.775,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 1,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.019230769230769232,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 1,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "liquid-d1-omni-2b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp16"
          },
          "timing": {
            "p50Ms": 18.5,
            "p95Ms": 23,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "text-only-input"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 49,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.8166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 11,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.9166666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 27,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.9,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 33,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.825,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 0,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "lyria-3-5",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "fal",
          "variant": "default",
          "expected": 4,
          "attempted": 4,
          "valid": 4,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "lyria-3-clip-preview",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "openrouter",
          "variant": "default",
          "expected": 4,
          "attempted": 4,
          "valid": 4,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "mai-transcribe-1-5",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 34,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.032692307692307694,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 75,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.012838069154399178,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "mai-transcribe-2",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 35,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.03365384615384615,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 75,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.012838069154399178,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "mai-transcribe-2-streaming",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "gateway",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 35,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "model-identity-limited",
            "peer-close-unverified",
            "limited-streaming",
            "retry-included"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 25,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.045454545454545456,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 11,
              "denominator": 461,
              "denominatorBasis": "reference-words",
              "value": 0.02386117136659436,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 36,
              "denominator": 1011,
              "denominatorBasis": "reference-words",
              "value": 0.03560830860534125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "stream-any-text",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "mai-voice-2",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 4,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.024844720496894408,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 9,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.010101010101010102,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 7,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.04375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 24,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.030226700251889168,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 11,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.03426791277258567,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 33,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.019584569732937686,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "mai-voice-2-1",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 12,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.037267080745341616,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 39,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.021885521885521887,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 10,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 40,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 22,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.03426791277258567,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 79,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.02344213649851632,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "mai-voice-2-1-flash",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 7,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.021739130434782608,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 28,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.015712682379349047,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 12,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.0375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 44,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.027707808564231738,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 19,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.029595015576323987,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 72,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.021364985163204748,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "mai-voice-2-flash",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 7,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.043478260869565216,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 33,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.037037037037037035,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 20,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 12,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.037383177570093455,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 53,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.0314540059347181,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "maya-2-native",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "local",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 14,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.0875,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 36,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.04534005037783375,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "minimax-music3",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "fal",
          "variant": "default",
          "expected": 4,
          "attempted": 4,
          "valid": 4,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "minimax-speech-2-8",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "hd",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 6,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.018633540372670808,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 9,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.005050505050505051,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 10,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 40,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 16,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.024922118380062305,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 49,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.014540059347181009,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        },
        {
          "id": "tts-proxy-2",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "turbo",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 2,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.006211180124223602,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 2,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.001122334455667789,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 10,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 40,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 12,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.018691588785046728,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 42,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.012462908011869436,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "moss-audio-4b-thinking",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "local",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "nonempty-output-only",
            "moss-token-limit-reached"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "muse-voice-transcribe",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 95,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.09134615384615384,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 313,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.05357754193769257,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "music-v2-5",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "fal",
          "variant": "default",
          "expected": 4,
          "attempted": 4,
          "valid": 4,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "nanojev",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 386.5,
            "p95Ms": 846,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 17,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.2833333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 1,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.08333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 10,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 8,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.2,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 20,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.38461538461538464,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 6,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "nemotron-3-diarization",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "local",
          "variant": "default",
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "nvidia-nemotron-3.5-transcribe",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 26,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.025,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 56,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.009585758301951387,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 4,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "orukeet",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 49,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.0890909090909091,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 70,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.02145922746781116,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 10,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.02040816326530612,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 21,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.00813953488372093,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 59,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.05673076923076923,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 91,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.015576857240671003,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "phonon-2",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 36,
          "attempted": 20,
          "valid": 20,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "language-support-unverified",
            "process-exit-unverified",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 12,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.024489795918367346,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 22,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.008527131782945736,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "en"
              ],
              "limitations": []
            }
          ]
        },
        {
          "id": "stt-2",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "language-support-unverified",
            "process-exit-unverified",
            "limited-language"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 171,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.3109090909090909,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 338,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.10361741263028816,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "pplx-decider-v1-27b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "bf16"
          },
          "timing": {
            "p50Ms": 1616.5,
            "p95Ms": 2101,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 58,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 29,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.9666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 40,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 0,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "pplx-decider-v1.1-27b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "bf16"
          },
          "timing": {
            "p50Ms": 1352.5,
            "p95Ms": 1785,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "text-only-input"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 58,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.9666666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 12,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 30,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 36,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.9,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 0,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 0,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "pyannote-community-1",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "cpu",
            "precision": "unknown"
          },
          "expected": 8,
          "attempted": 8,
          "valid": 8,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "synthetic-speaker-boundaries"
          ],
          "metrics": [
            {
              "kind": "technical-diarization-error",
              "numerator": 67.32206249999996,
              "denominator": 426.41700000000003,
              "denominatorBasis": "reference-speaker-time",
              "value": 0.15787846755640594,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "technical-only",
                "synthetic-speaker-boundaries",
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "scribe-v2",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "fal",
          "variant": "default",
          "timing": {
            "p50Ms": 2745,
            "p95Ms": 6215,
            "sampleCount": 36,
            "sampleUnit": "cases",
            "scope": "hosted-end-to-end",
            "quantileMethod": "median-and-nearest-rank"
          },
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 20,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.03636363636363636,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 32,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.009809932556713672,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 8,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.0163265306122449,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 16,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.006201550387596899,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 28,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.026923076923076925,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 48,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.008216364258815475,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "scribe-v2-medical",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 26,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.025,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 45,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.007702841492639507,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "scribe-v2-realtime",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "gateway",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited",
            "peer-close-unverified",
            "limited-streaming"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 39,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.07090909090909091,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 9,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.018367346938775512,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 48,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.046153846153846156,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "stream-any-text",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "strands-decider-2b",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "fp32"
          },
          "timing": {
            "p50Ms": 175.5,
            "p95Ms": 623,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-gpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "base-revision-unattested"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 32,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.5333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 10,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.8333333333333334,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 9,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.75,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 20,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.6666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 21,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.525,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 9,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.17307692307692307,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 1,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.125,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "universal-3-6-pro-realtime",
      "category": "audio",
      "status": "partial",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "livekit",
          "variant": "default",
          "expected": 108,
          "attempted": 108,
          "valid": 96,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "unreviewed-reference",
            "model-identity-limited",
            "no-silence-assessment",
            "limited-streaming",
            "repeated-inputs",
            "read-speech"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 121,
              "denominator": 1650,
              "denominatorBasis": "reference-words",
              "value": 0.07333333333333333,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 222,
              "denominator": 9786,
              "denominatorBasis": "reference-characters",
              "value": 0.022685469037400367,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 24,
              "denominator": 1470,
              "denominatorBasis": "reference-words",
              "value": 0.0163265306122449,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 63,
              "denominator": 7740,
              "denominatorBasis": "reference-characters",
              "value": 0.00813953488372093,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 145,
              "denominator": 3120,
              "denominatorBasis": "reference-words",
              "value": 0.046474358974358976,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 285,
              "denominator": 17526,
              "denominatorBasis": "reference-characters",
              "value": 0.01626155426223896,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "numerator": 0,
              "denominator": 0,
              "denominatorBasis": "assessable-silence",
              "value": null,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "no-silence-assessment"
              ]
            }
          ]
        },
        {
          "id": "technical-2",
          "protocol": "technical",
          "route": "livekit",
          "variant": "default",
          "expected": 1,
          "attempted": 1,
          "valid": 1,
          "languages": [
            "en"
          ],
          "limitations": [
            "partial-coverage",
            "smoke-only",
            "no-quality-assessment",
            "technical-only",
            "limited-inputs",
            "model-identity-limited",
            "no-silence-assessment",
            "limited-streaming",
            "repeated-inputs",
            "read-speech",
            "limited-language"
          ],
          "metrics": []
        }
      ]
    },
    {
      "modelId": "von-1.1",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 413.5,
            "p95Ms": 782,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 23,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.38333333333333336,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 5,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.4166666666666667,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 4,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 12,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.4,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 18,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.45,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 4,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.07692307692307693,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 8,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 1,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "von-1.2",
      "category": "decision",
      "status": "complete",
      "cohorts": [
        {
          "id": "decision",
          "protocol": "decision",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "unknown",
            "precision": "unknown"
          },
          "timing": {
            "p50Ms": 136.5,
            "p95Ms": 218,
            "sampleCount": 130,
            "sampleUnit": "cases",
            "scope": "local-cpu",
            "quantileMethod": "unknown"
          },
          "expected": 130,
          "attempted": 130,
          "valid": 130,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "decision-accuracy",
              "numerator": 20,
              "denominator": 60,
              "denominatorBasis": "planned-cases",
              "value": 0.3333333333333333,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "triage",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "routing",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "constraints",
              "numerator": 2,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.16666666666666666,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "tables",
              "numerator": 3,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.25,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-accuracy",
              "family": "evidence",
              "numerator": 6,
              "denominator": 12,
              "denominatorBasis": "planned-cases",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-robustness-accuracy",
              "numerator": 6,
              "denominator": 30,
              "denominatorBasis": "planned-cases",
              "value": 0.2,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-development-accuracy",
              "numerator": 14,
              "denominator": 40,
              "denominatorBasis": "planned-cases",
              "value": 0.35,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-false-review-rate",
              "numerator": 23,
              "denominator": 52,
              "denominatorBasis": "negative-cases",
              "value": 0.4423076923076923,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "decision-missed-review-rate",
              "numerator": 3,
              "denominator": 8,
              "denominatorBasis": "positive-cases",
              "value": 0.375,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "voxtral-mini-transcribe-2",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 51,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.04903846153846154,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 99,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.016946251283806914,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 2,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0.5,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "voxtral-mini-transcribe-3-0",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 33,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.03173076923076923,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 78,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.013351591920575145,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "voxtral-mini-transcribe-realtime-2602",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "local",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 57,
              "denominator": 550,
              "denominatorBasis": "reference-words",
              "value": 0.10363636363636364,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 146,
              "denominator": 3262,
              "denominatorBasis": "reference-characters",
              "value": 0.04475781729000613,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 11,
              "denominator": 490,
              "denominatorBasis": "reference-words",
              "value": 0.022448979591836733,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 27,
              "denominator": 2580,
              "denominatorBasis": "reference-characters",
              "value": 0.010465116279069767,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-wer",
              "numerator": 68,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.06538461538461539,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 173,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.029613146182814104,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "voxtral-mini-tts-2603",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 16,
          "attempted": 16,
          "valid": 16,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 10,
              "denominator": 161,
              "denominatorBasis": "reference-words",
              "value": 0.062111801242236024,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 27,
              "denominator": 891,
              "denominatorBasis": "reference-characters",
              "value": 0.030303030303030304,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 5,
              "denominator": 160,
              "denominatorBasis": "reference-words",
              "value": 0.03125,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 20,
              "denominator": 794,
              "denominatorBasis": "reference-characters",
              "value": 0.02518891687657431,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 15,
              "denominator": 321,
              "denominatorBasis": "reference-words",
              "value": 0.04672897196261682,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 47,
              "denominator": 1685,
              "denominatorBasis": "reference-characters",
              "value": 0.027893175074183978,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "voxtral-mini-tts-3-0",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "tts-proxy-1",
          "protocol": "tts-proxy",
          "route": "openrouter",
          "variant": "default",
          "expected": 32,
          "attempted": 32,
          "valid": 32,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "asr-proxy",
            "no-naturalness-evaluation",
            "provider-voice-unverified",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "tts-asr-wer",
              "numerator": 15,
              "denominator": 322,
              "denominatorBasis": "reference-words",
              "value": 0.046583850931677016,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 50,
              "denominator": 1782,
              "denominatorBasis": "reference-characters",
              "value": 0.028058361391694726,
              "languages": [
                "de"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 18,
              "denominator": 320,
              "denominatorBasis": "reference-words",
              "value": 0.05625,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 46,
              "denominator": 1588,
              "denominatorBasis": "reference-characters",
              "value": 0.028967254408060455,
              "languages": [
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-wer",
              "numerator": 33,
              "denominator": 642,
              "denominatorBasis": "reference-words",
              "value": 0.0514018691588785,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            },
            {
              "kind": "tts-asr-cer",
              "numerator": 96,
              "denominator": 3370,
              "denominatorBasis": "reference-characters",
              "value": 0.028486646884272996,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference",
                "asr-proxy",
                "no-naturalness-evaluation"
              ]
            }
          ]
        }
      ]
    },
    {
      "modelId": "voxtral-small-transcribe-3-0",
      "category": "audio",
      "status": "complete",
      "cohorts": [
        {
          "id": "stt-1",
          "protocol": "stt",
          "route": "openrouter",
          "variant": "default",
          "expected": 36,
          "attempted": 36,
          "valid": 36,
          "languages": [
            "de",
            "en"
          ],
          "limitations": [
            "unreviewed-reference",
            "model-identity-limited"
          ],
          "metrics": [
            {
              "kind": "stt-wer",
              "numerator": 33,
              "denominator": 1040,
              "denominatorBasis": "reference-words",
              "value": 0.03173076923076923,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-cer",
              "numerator": 72,
              "denominator": 5842,
              "denominatorBasis": "reference-characters",
              "value": 0.012324546388223211,
              "languages": [
                "de",
                "en"
              ],
              "limitations": [
                "unreviewed-reference"
              ]
            },
            {
              "kind": "stt-silence-text",
              "silenceAssessment": "final-normalized-words",
              "numerator": 0,
              "denominator": 4,
              "denominatorBasis": "assessable-silence",
              "value": 0,
              "languages": [
                "de",
                "en"
              ],
              "limitations": []
            }
          ]
        }
      ]
    },
    {
      "modelId": "yue2-3b",
      "category": "audio",
      "status": "technical",
      "cohorts": [
        {
          "id": "technical-1",
          "protocol": "technical",
          "route": "local",
          "variant": "default",
          "execution": {
            "runtime": "apple-metal",
            "precision": "q4_0",
            "decoderPrecision": "fp16"
          },
          "expected": 2,
          "attempted": 2,
          "valid": 2,
          "languages": [
            "unknown"
          ],
          "limitations": [
            "technical-only",
            "limited-inputs",
            "quantized-execution",
            "noncommercial-weights",
            "no-quality-assessment"
          ],
          "metrics": []
        }
      ]
    }
  ]
}
